Serving Configuration & Startup Params

feed_var {
  name: "x"
  alias_name: "x"
  is_lod_tensor: false
  feed_type: 1
  shape: 13
}
fetch_var {
  name: "concat_1.tmp_0"
  alias_name: "concat_1.tmp_0"
  is_lod_tensor: false
  fetch_type: 1
  shape: 3
  shape: 640
  shape: 640
}

python3 -m paddle_serving_server.serve --model serving_model --port 9393

workdir_9393
├── general_infer_0
│   ├── fluid_time_file
│   ├── general_model.prototxt
│   └── model_toolkit.prototxt
├── infer_service.prototxt
├── resource.prototxt
└── workflow.prototxt

python3 -m paddle_serving_server.serve --model serving_model --thread 10 --port 9292 --gpu_ids 0,1,2

python3 -m paddle_serving_server.serve --model serving_model_1 serving_model_2 --thread 10 --port 9292

python3 -m paddle_serving_server.serve stop

/bin/serving --flagfile=proj.conf

# for paddle inference
--precision=fp32
--use_calib=False
--reload_interval_s=10
# for brpc
--max_concurrency=0
--num_threads=10
--bthread_concurrency=10
--max_body_size=536870912
# default path
--inferservice_path=conf
--inferservice_file=infer_service.prototxt
--resource_path=conf
--resource_file=resource.prototxt
--workflow_path=conf
--workflow_file=workflow.prototxt

port: 8010
services {
  name: "GeneralModelService"
  workflows: "workflow1"
}

workflows {
  name: "workflow1"
  workflow_type: "Sequence"
  nodes {
    name: "general_reader_0"
    type: "GeneralReaderOp"
  }
  nodes {
    name: "general_infer_0"
    type: "GeneralInferOp"
    dependencies {
      name: "general_reader_0"
      mode: "RO"
    }
  }
  nodes {
    name: "general_response_0"
    type: "GeneralResponseOp"
    dependencies {
      name: "general_infer_0"
      mode: "RO"
    }
  }
}

model_toolkit_path: "conf"
model_toolkit_file: "general_infer_0/model_toolkit.prototxt"
general_model_path: "conf"
general_model_file: "general_infer_0/general_model.prototxt"

engines {
  name: "general_infer_0"
  type: "PADDLE_INFER"
  reloadable_meta: "uci_housing_model/fluid_time_file"
  reloadable_type: "timestamp_ne"
  model_dir: "uci_housing_model"
  gpu_ids: -1
  enable_memory_optimization: true
  enable_ir_optimization: false
  use_trt: false
  use_lite: false
  use_xpu: false
  use_gpu: false
  combined_model: false
  gpu_multi_stream: false
  use_ascend_cl: false
  runtime_thread_num: 0
  batch_infer_size: 32
  enable_overrun: false
  allow_split_request: true
}

# GPU0，1，2
gpu_ids: 0
gpu_ids: 1
gpu_ids: 2

feed_var {
  name: "x"
  alias_name: "x"
  is_lod_tensor: false
  feed_type: 1
  shape: 13
}
fetch_var {
  name: "fc_0.tmp_1"
  alias_name: "price"
  is_lod_tensor: false
  fetch_type: 1
  shape: 1
}

python3 web_service.py

python3 -m paddle_serving_server.serve stop

#RPC port. The RPC port and HTTP port cannot be empyt at the same time. If the RPC port is empty and the HTTP port is not empty, the RPC port is automatically set to HTTP port+1.
rpc_port: 18090

#HTTP port. The RPC port and the HTTP port cannot be empty at the same time. If the RPC port is available and the HTTP port is empty, the HTTP port is not automatically generated
http_port: 9999

#worker_num, the maximum concurrency.
#When build_dag_each_worker=True, server will create processes within GRPC Server ans DAG.
#When build_dag_each_worker=False, server will set the threadpool of GRPC.
worker_num: 20

#build_dag_each_worker, False，create process with DAG；True，create process with multiple independent DAG
build_dag_each_worker: false

dag:
    #True, thread model；False，process model
    is_thread_op: False

    #retry times
    retry: 1

    # True，generate the TimeLine data；False
    use_profile: false
    tracer:
        interval_s: 10

    #client type，include brpc, grpc and local_predictor.
    #client_type: local_predictor

    # max channel size, default 0
    #channel_size: 0

    #For distributed large model scenario with tensor parallelism, the first result is received and the other results are discarded to provide speed
    #channel_recv_frist_arrive: False

op:
    det:
        #concurrency，is_thread_op=True，thread otherwise process
        concurrency: 6

        #Serving IPs
        #server_endpoints: ["127.0.0.1:9393"]

        #Fetch data list
        #fetch_list: ["concat_1.tmp_0"]

        #det client config
        #client_config: serving_client_conf.prototxt

        #Serving timeout, ms
        #timeout: 3000

        #Serving retry times
        #retry: 1

        #Default 1。batch_size>1 should set auto_batching_timeout
        #batch_size: 2

        #Batching timeout，used with batch_size
        #auto_batching_timeout: 2000

        #Loading local server configuration without server_endpoints.
        local_service_conf:
            #client type，include brpc, grpc and local_predictor.
            client_type: local_predictor

            #det model path
            model_config: ocr_det_model

            #Fetch data list
            fetch_list: ["concat_1.tmp_0"]

            # device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu, 5=arm ascend310, 6=arm ascend910
            device_type: 0

            #Device ID
            devices: ""

            #use_mkldnn, When running on mkldnn，must set ir_optim=True
            #use_mkldnn: True

            #ir_optim, When running on TensorRT，must set ir_optim=True
            ir_optim: True
            
            #CPU 计算线程数，在CPU场景开启会降低单次请求响应时长
            #thread_num: 10
            
            #precsion, Decrease accuracy can increase speed
            #GPU 支持: "fp32"(default), "fp16", "int8"；
            #CPU 支持: "fp32"(default), "fp16", "bf16"(mkldnn); 不支持: "int8"
            precision: "fp32"

            #mem_optim, memory / graphic memory optimization
            #mem_optim: True

            #use_calib, Use TRT int8 calibration
            #use_calib: False

            #use_mkldnn, Use mkldnn for cpu
            #use_mkldnn: False

            #The cache capacity of different input shapes for mkldnn
            #mkldnn_cache_capacity: 0

            #mkldnn_op_list, op list accelerated using MKLDNN, None default
            #mkldnn_op_list: []

            #mkldnn_bf16_op_list,op list accelerated using MKLDNN bf16, None default.
            #mkldnn_bf16_op_list: []

            #min_subgraph_size,the minimal subgraph size for opening tensorrt to optimize, 3 default
            #min_subgraph_size: 3
    rec:
        #concurrency，is_thread_op=True，thread otherwise process
        concurrency: 3

        #time out, ms
        timeout: -1

        #retry times
        retry: 1

        #Loading local server configuration without server_endpoints.
        local_service_conf:

            #client type，include brpc, grpc and local_predictor.
            client_type: local_predictor

            #rec model path
            model_config: ocr_rec_model

            #Fetch data list
            fetch_list: ["ctc_greedy_decoder_0.tmp_0", "softmax_0.tmp_0"]

            # device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu, 5=arm ascend310, 6=arm ascend910
            device_type: 0
            
            #Device ID
            devices: ""

            #use_mkldnn, When running on mkldnn，must set ir_optim=True
            #use_mkldnn: True

            #ir_optim, When running on TensorRT，must set ir_optim=True
            ir_optim: True
            
            #CPU 计算线程数，在CPU场景开启会降低单次请求响应时长
            #thread_num: 10
            
            #precsion, Decrease accuracy can increase speed
            #GPU 支持: "fp32"(default), "fp16", "int8"；
            #CPU 支持: "fp32"(default), "fp16", "bf16"(mkldnn); 不支持: "int8"
            precision: "fp32"

#True, thread model；False，process model
is_thread_op: False

#concurrency，is_thread_op=True，thread otherwise process
concurrency: 7

devices: "0,1,2"

# device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu, 5=arm ascend310, 6=arm ascend910
device_type: 0
devices: "" # "0,1"

#precsion
#GPU support: "fp32"(default), "fp16(TensorRT)", "int8"；
#CPU support: "fp32"(default), "fp16", "bf16"(mkldnn); not support: "int8"
precision: "fp32"

#cablic, open it when using int8
use_calib: True