目录

tag_match

阿里云

182.92.154.119

lsof -i :5000

tail -f 文件路径

Conda环境

flask

requests

flask_restful

flask_cors

numpy

sentence_transformers==2.2.2

milvus

pandas

xlsxwriter

openai

pymilvus

openpyxl

zhipuai

json

sacremoses

  • 拷贝环境

conda pack -n NLP -o NLP.tar.gz

部署

milvus

cpu

wget https://github.com/milvus-io/milvus/releases/download/v2.3.2/milvus-standalone-docker-compose.yml -O docker-compose.yml sudo docker-compose up -d

gpu

wget https://github.com/milvus-io/milvus/releases/download/v2.3.2/milvus-standalone-docker-compose-gpu.yml -O docker-compose.yml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
standalone:
  environment:
    ...
    KNOWHERE_GPU_MEM_POOL_SIZE: 2048;4096
  ...
  deploy:
    resources:
      reservations:
        devices:
          - driver: nvidia
            capabilities: ["gpu"]
            device_ids: ['0', '1']

sudo docker compose up -d

开启用户

docker文件里的config

attu

  • 端口 8000 docker run -p 8000:3000 -d zilliz/attu:latest

neo4j

安装

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
# 在/usr/local下创建java文件夹,将下载的jdk上传至java文件夹下
mkdir /usr/local/java
# 解压上传的jdk文件夹到当前目录下
tar -zxvf jdk-11.0.18_linux-x64_bin.tar.gz

# 在/usr/local/下创建neo4j文件夹,
mkdir /usr/local/neo4j
# 上传neo4j压缩包至该文件夹,并解压
tar -zxvf neo4j-community-4.4.18-unix.tar.gz

# 修改环境变量
vim /etc/profile

#Java环境变量配置
export JAVA_HOME="/usr/local/java/jdk-11.0.18"
export PATH="$JAVA_HOME/bin:$PATH"
export CLASSPATH=".:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar"

#neo4j环境变量配置
PATH="/usr/local/neo4j/neo4j-community-4.4.18/bin:$PATH"

# 更新环境变量文件,使修改立即生效
source /etc/profile
# 查看jdk配置是否正确
java -version

# 修改neo4j配置文件,让它可以从`远程访问控制台`
vim /usr/local/neo4j/neo4j-community-4.4.18/conf/neo4j.conf

# 查看防火墙状态
systemctl status firewalld
# 关闭防火墙
systemctl stop firewalld

neo4j.conf修改内容如下:
取消注释的  dbms.default_listen_address=0.0.0.0

# 启动neo4j;
neo4j start
# 登录用户:neo4j    登录密码:neo4j

tag_match.assets/a1785ea6b5904f5ea7f87104aebf94d4.png

使用文档

使用neo4j数据库

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
from py2neo import Graph, Node

# 建立连接
graph = Graph("bolt://182.92.154.119:7687", auth=("neo4j", "123456789"))

# 执行查询
result = graph.run("SHOW DATABASES")

# 处理结果
for record in result:
    print(record)

使用system数据库

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
from py2neo import SystemGraph, Node

# 建立连接
system_graph = SystemGraph("bolt://182.92.154.119:7687", auth=("neo4j", "123456789"))

# 创建节点
person = Node("Person", name="John")
city = Node("City", name="New York")

# 将节点添加到图中
system_graph.create(person)
system_graph.create(city)

# 创建节点之间的关系
knows = person.relationships.create("KNOWS", city)

# 将关系添加到图中
system_graph.create(knows)

ChatGLM

2:

  • "请猜测 'id' 字段的中文含义,该字段在 bjnd 数据库的 auth_role 表中,长度为20个字符,数据类型为 bigint 类型。回复内容5个字以内",
    
    "\"权限角色\""
    
  • “请猜测 ‘chkpwdtime’ 字段的中文含义,该字段在 bjnd 数据库的 user表中,长度为10个字符,数据类型为 int类型。回复内容5个字以内”,

  • {“prompt”: “请猜测 ‘chkpwdtime’ 字段的中文含义,该字段在 bjnd 数据库的 user表中,长度为10个字符,数据类型为 int类型。回复内容5个字以内”, “history”: [[“请猜测 ‘id’ 字段的中文含义,该字段在 bjnd 数据库的 auth_role 表中,长度为20个字符,数据类型为 bigint 类型。回复内容5个字以内”, “权限ID”]]}

  • data = {

    “prompt”: query_str,

    “history”: [[“请猜测 ‘id’ 字段的中文含义,该字段在 bjnd 数据库的 auth_role 表中,长度为20个字符,数据类型为 bigint 类型。回复内容5个字以内”, “权限ID”]]

    }

3:

https://github.com/THUDM/ChatGLM3/blob/main/PROMPT.md

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
    response =  openai.ChatCompletion.create(
    model="chatglm3-6b",
    messages=[
            {"role": "user", "content": "请猜测 'id' 字段的中文含义,该字段在 bjnd 数据库的 auth_role 表中,长度为20个字符,数据类型为 bigint 类型。回复内容5个字以内"},
            {"role": "assistant", "content": "权限ID"},
            {"role": "user", "content": query_str}
    ],
    stream=False
    )
    content = response["choices"][0]["message"]["content"]

tag_match.assets/image-20231205191157225.png

Docker

https://download.docker.com/

离线安装

https://blog.csdn.net/MssGuo/article/details/122694156

安装docker

  • 下载文件

https://download.docker.com/linux/static/stable/x86_64/

docker-24.0.7.tgz

  • 安装

    1
    2
    3
    
    cd tag_match
    tar  -xf docker-24.0.7.tgz
    cd docker	&& cp * /usr/bin/
    
  • 将docker做成系统服务,并设置开机自启

    #创建一个docker.service文件,并放到/usr/lib/systemd/system/目录下 #注意,这个docker。server文件是从官方复制过来的,但是删除了一些东西,如下 #Wants=network-online.target #注意:这里删除了Requires=docker.socket containerd.service 这一条,不然会启动失败 #ExecStart=/usr/bin/dockerd #注意:这里删除了-H fd:// –containerd=/run/containerd/containerd.sock这行不然会启动失败

     1
     2
     3
     4
     5
     6
     7
     8
     9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    
    cat > /usr/lib/systemd/system/docker.service <<'EOF'
    [Unit]
    Description=Docker Application Container Engine
    Documentation=https://docs.docker.com
    After=network-online.target firewalld.service containerd.service
    Wants=network-online.target
    
    [Service]
    Type=notify
    # the default is not to use systemd for cgroups because the delegate issues still
    # exists and systemd currently does not support the cgroup feature set required
    # for containers run by docker
    ExecStart=/usr/bin/dockerd
    ExecReload=/bin/kill -s HUP $MAINPID
    TimeoutSec=0
    RestartSec=2
    Restart=always
    
    # Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
    # Both the old, and new location are accepted by systemd 229 and up, so using the old location
    # to make them work for either version of systemd.
    StartLimitBurst=3
    
    # Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
    # Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
    # this option work for either version of systemd.
    StartLimitInterval=60s
    
    # Having non-zero Limit*s causes performance problems due to accounting overhead
    # in the kernel. We recommend using cgroups to do container-local accounting.
    LimitNOFILE=infinity
    LimitNPROC=infinity
    LimitCORE=infinity
    
    # Comment TasksMax if your systemd version does not support it.
    # Only systemd 226 and above support this option.
    TasksMax=infinity
    
    # set delegate yes so that systemd does not reset the cgroups of docker containers
    Delegate=yes
    
    # kill only the docker process, not all processes in the cgroup
    KillMode=process
    OOMScoreAdjust=-500
    
    [Install]
    WantedBy=multi-user.target
    EOF
    
    1
    2
    3
    4
    
    systemctl daemon-reload
    systemctl  start  docker
    systemctl  status docker
    systemctl  enable docker
    

安装docker compose

  • 下载文件

    1
    
    https://github.com/docker/compose
    

    docker-compose

  • 移动

    1
    2
    
    cd tag_match
    cp docker-compose /usr/local/bin/docker-compose
    
  • 授予执行权限

    1
    2
    
    chmod a+x /usr/local/bin/docker-compose
    docker-compose -v
    

https://www.jb51.net/article/260369.htm

离线安装Attu

  • 导出镜像

    docker images

    docker save -o attu.tar zilliz/attu:v2.3.1

  • 加载镜像

    docker load < attu.tar

  • 查看镜像

    docker images

  • 使用镜像启动容器实例

    docker run -p 8000:3000 -d zilliz/attu:v2.3.1

离线安装Milvus

  • 导入镜像

    cd tag_match

    docker load < attu.tar && docker load < minio.tar && docker load < quay.tar && docker load < milvusdb.tar

  • 创建文件夹

    cd .. && mkdir milvus && cd milvus

  • 启动milvus组件

    cp ../tag_match/docker-compose.yml ./

    docker-compose up -d

  • 启动attu

    docker run -p 8000:3000 -d zilliz/attu:v2.3.1

mkdir /root/miniconda3/envs/tag

tar -xvf file.tar.gz -C /path/to/target/directory

一键部署脚本

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
#!/bin/bash

# 作者: wjm
# 版本: 1.0
# 创建日期: 2024-01-10
# 描述: 这是一个运行 Python 脚本的 Bash 脚本,激活 conda 环境并在后台运行脚本。

echo "╔════════════════════════╗"
echo "║     你的脚本开始啦     ║"
echo "╚════════════════════════╝"

# 检查当前用户是否为root
if [ "$(whoami)" != "root" ]; then
    echo "需要root权限运行该脚本,请使用sudo或切换到root用户。"
    exit 1
fi

echo "安装mniconda,默认位置:/root/miniconda3"
echo "请根据信息完成安装"
./Miniconda3-py311_23.11.0-1-Linux-x86_64.sh

eval "$(/root/miniconda3/bin/conda shell.bash hook)"
echo "mniconda安装完成"

echo "开始拷贝tag环境"
mkdir /root/miniconda3/envs/tag
tar -xvf tag.tar.gz -C /root/miniconda3/envs/tag
echo "拷贝tag环境完成"

echo "开始安装 Docker..."

# 安装 Docker
tar -xvf docker-24.0.7.tgz
if [ $? -eq 0 ]; then
    echo "Docker安装成功"
else
    echo "Docker安装失败,请检查错误并解决问题"
    exit 1
fi

cp docker/* /usr/bin/

echo "设置 Docker 服务..."

# 将 Docker 设置为系统服务,并设置开机自启
cat > /usr/lib/systemd/system/docker.service <<'EOF'
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service containerd.service
Wants=network-online.target

[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3

# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new location are accepted by systemd 230 and up, so using the old name
# to make this option work for either version of systemd.
StartLimitInterval=60s

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMax=infinity

# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes

# kill only the docker process, not all processes in the cgroup
KillMode=process
OOMScoreAdjust=-500

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl start docker
if [ $? -eq 0 ]; then
    echo "Docker服务已启动"
else
    echo "Docker服务启动失败,请检查错误并解决问题"
    exit 1
fi

systemctl status docker
echo "Docker服务已设置为开机自启"
systemctl enable docker

echo "安装 Docker Compose..."

# 安装 Docker Compose
cp docker-compose /usr/local/bin/docker-compose
if [ $? -eq 0 ]; then
    echo "Docker Compose安装成功"
else
    echo "Docker Compose安装失败,请检查错误并解决问题"
    exit 1
fi

chmod a+x /usr/local/bin/docker-compose
docker-compose -v

echo "开始离线安装 Milvus..."

# 离线安装 Milvus
mkdir milvus
cd milvus

# 导入镜像
docker load < ../attu.tar && docker load < ../minio.tar && docker load < ../quay.tar && docker load < ../milvusdb.tar
if [ $? -eq 0 ]; then
    echo "Milvus镜像已导入"
else
    echo "Milvus镜像导入失败,请检查错误并解决问题"
    exit 1
fi

# 启动 Milvus 组件
cp ../docker-compose.yml ./
docker-compose up -d
if [ $? -eq 0 ]; then
    echo "Milvus组件已启动"
else
    echo "Milvus组件启动失败,请检查错误并解决问题"
    exit 1
fi

echo "启动 Attu..."

# 启动 Attu
docker run -p 8000:3000 -d zilliz/attu:v2.3.1
if [ $? -eq 0 ]; then
    echo "Attu已启动"
else
    echo "Attu启动失败,请检查错误并解决问题"
    exit 1
fi

eval "$(/root/miniconda3/bin/conda shell.bash hook)"

echo "所有步骤已完成!"

echo "╔════════════════════════╗"
echo "║    你的脚本执行完成    ║"
echo "╚════════════════════════╝"

一键启动项目

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
#!/bin/bash

# 作者: wjm
# 版本: 1.0
# 创建日期: 2024-01-10
# 描述: 这是一个运行 Python 脚本的 Bash 脚本,激活 conda 环境并在后台运行脚本。

echo "╔════════════════════════╗"
echo "║     你的脚本开始啦     ║"
echo "╚════════════════════════╝"

# 检查当前用户是否为root
if [ "$(whoami)" != "root" ]; then
    echo "需要root权限运行该脚本,请使用sudo或切换到root用户。"
    exit 1
fi


eval "$(/root/miniconda3/bin/conda shell.bash hook)"

echo "启动容器"
# 获取所有运行中的容器 ID
container_ids=$(docker ps -q)

# 遍历容器 ID 并启动每个容器
for container_id in $container_ids; do
    echo "启动容器: $container_id"
    docker start $container_id
done

echo "所有容器已启动"

conda env list
echo "激活 conda 环境..."
conda activate tag

if [ $? -ne 0 ]; then
    echo "激活 conda 环境失败,请检查错误并解决问题"
    exit 1
fi

echo "后台运行 Python 程序..."
cd code
nohup python milvus.py

if [ $? -eq 0 ]; then
    echo "Python 程序已在后台运行"
else
    echo "启动 Python 脚本失败,请检查错误并解决问题"
    exit 1
fi

echo "╔════════════════════════╗"
echo "║    你的脚本执行完成    ║"
echo "╚════════════════════════╝"

Flask

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
返回Json
from flask import Flask

app = Flask(__name__)

@app.route("/")
def index():
    response = flask.Response()
    response.set_data(json.dumps({"name": "John Doe"}))
    response.headers["Content-Type"] = "application/json"
    return response

if __name__ == "__main__":
    app.run()

模型

ebd:

sentence-transformers/distiluse-base-multilingual-cased-v2

translate:

zhijian12345/marian-finetuned-kde4-en-to-zh_CN

配置日志

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
import logging
from logging.handlers import RotatingFileHandler
# 配置日志记录器
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(filename)s - %(levelname)s - %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
    filename='app.log',  # 日志文件名
    filemode='a',       # 追加模式
    encoding='utf-8'     # 指定编码为 utf-8
)

# 创建一个RotatingFileHandler,用于日志文件的轮转
handler = RotatingFileHandler(
    'app.log', 
    maxBytes=10000000, 
    backupCount=5
)

# 创建一个日志格式器
formatter = logging.Formatter(
    '%(asctime)s - %(filename)s - %(levelname)s - %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S'
)

# 为RotatingFileHandler设置格式器
handler.setFormatter(formatter)

# 获取当前模块的日志记录器
logger = logging.getLogger(__name__)

# 检查是否已经存在 handler,避免重复添加
if not logger.hasHandlers():
    logger.addHandler(handler)

Milvus 2.4部署方式

wget https://raw.githubusercontent.com/milvus-io/milvus/master/scripts/standalone_embed.sh bash standalone_embed.sh start

bash standalone_embed.sh stop

bash standalone_embed.sh delete