{"id":14469,"date":"2023-02-26T22:24:52","date_gmt":"2023-02-26T14:24:52","guid":{"rendered":"http:\/\/139.9.1.231\/?p=14469"},"modified":"2024-12-31T10:52:53","modified_gmt":"2024-12-31T02:52:53","slug":"tensorrt-7","status":"publish","type":"post","link":"http:\/\/139.9.1.231\/index.php\/2023\/02\/26\/tensorrt-7\/","title":{"rendered":"TensorRT 7 \u52a8\u6001\u8f93\u5165\u548c\u8f93\u51fa"},"content":{"rendered":"\n

\u6559\u7a0b\u6587\u6863\uff1a<\/strong><\/p>\n\n\n\n

\u30101\u3011https:\/\/docs.nvidia.com\/deeplearning\/tensorrt\/index.html<\/a><\/p>\n\n\n\n

\u30102\u3011https:\/\/developer.nvidia.com\/zh-cn\/tensorrt<\/a><\/p>\n\n\n\n

\u30103\u3011Nvidia TensorRT\u6587\u6863\u2014\u2014\u5f00\u53d1\u8005\u6307\u5357<\/a><\/p>\n\n\n\n

\u30104\u3011github: https:\/\/github.com\/NVIDIA\/TensorRT<\/a><\/p>\n\n\n\n

<\/strong> [5]<\/strong> pytorch-onnx-tensorrt\u5168\u94fe\u8def\u7b80\u5355\u6559\u7a0b(\u652f\u6301\u52a8\u6001\u8f93\u5165)<\/a><\/p>\n\n\n\n

[6] Pytorch\u8f6connx, TensorRT\u8e29\u5751\u5b9e\u5f55<\/a><\/strong><\/p>\n\n\n\n

\u6d41\u7a0b\uff1a<\/strong><\/h4>\n\n\n\n

\uff081\uff09Torch\u6a21\u578b\u8f6connx<\/u><\/strong><\/p>\n\n\n\n

\uff082\uff09Onnx\u8f6cTensorRT engine\u6587\u4ef6<\/u><\/strong><\/p>\n\n\n\n

\uff083\uff09TensorRT\u52a0\u8f7dengine\u6587\u4ef6\u5e76\u5b9e\u73b0\u63a8\u7406<\/u><\/strong><\/p>\n\n\n\n

Torch\u6a21\u578b\u8f6connx<\/strong>\uff1a<\/p>\n\n\n\n

def torch_2_onnx(model, MODEL_ONNX_PATH ):\n    OPERATOR_EXPORT_TYPE = torch._C._onnx.OperatorExportTypes.ONNX\n    \"\"\"\n    \u8fd9\u91cc\u6784\u5efa\u7f51\u7edc\u7684\u8f93\u5165\uff0c\u6709\u51e0\u4e2a\u5c31\u6784\u5efa\u51e0\u4e2a\n    \u548c\u7f51\u7edc\u6b63\u5e38\u7684inference\u65f6\u8f93\u5165\u4e00\u81f4\u5c31\u53ef\u4ee5\n    \"\"\"\n    org_dummy_input = (inputs_1, inputs_2, inputs_3, inputs_4)\n \n    #\u8fd9\u662f\u652f\u6301\u52a8\u6001\u8f93\u5165\u548c\u8f93\u51fa\u7684\u7b2c\u4e00\u6b65\n    #\u6bcf\u4e00\u4e2a\u8f93\u5165\u548c\u8f93\u51fa\u4e2d\u52a8\u6001\u7684\u7ef4\u5ea6\u90fd\u8981\u5728\u6b64\u6ce8\u660e\uff0c\u4e0b\u6807\u4ece0\u5f00\u59cb\n    dynamic_axes = {\n                        'inputs_1': {0:'batch_size', 1:'text_length'},\n                        'inputs_2': {0:'batch_size', 1:'text_length'},\n                        'inputs_3': {0:'batch_size', 1:'text_length'},\n                        'inputs_4': {0:'batch_size', 1:'text_length'},\n                        'outputs': {0:'batch_size', 1:'text_length'},\n                        }\n    output = torch.onnx.export( model,\n                                org_dummy_input,\n                                MODEL_ONNX_PATH,\n                                verbose=True,\n                                opset_version=11,\n                                operator_export_type=OPERATOR_EXPORT_TYPE,\n                                input_names=['inputs_1', 'inputs_2', 'inputs_3', 'inputs_4'],\n                                    output_names=['outputs'],\n                                    dynamic_axes=dynamic_axes\n                                )\n    print(\"Export of model to {}\".format(MODEL_ONNX_PATH))\n\n\n\u4f7f\u7528onnxruntime\u8fdb\u884c\u9a8c\u8bc1<\/strong>\n\n1. \u5b89\u88c5onnxruntime\n\npip install onnxruntime\n2. \u4f7f\u7528onnxruntime\u9a8c\u8bc1\n\nimport onnxruntime as ort\n\t\nort_session = ort.InferenceSession('xxx.onnx')\nimg = cv2.imread('test.jpg')\nnet_input = preprocess(img) # \u4f60\u7684\u9884\u5904\u7406\u51fd\u6570\noutputs = ort_session.run(None, {ort_session.get_inputs()[0].name: net_input})\nprint(outputs)\n\n3. \u4e0a\u9762\u7684\u8fd9\u6bb5\u4ee3\u7801\u6709\u4e00\u70b9\u9700\u8981\u6ce8\u610f\uff0c\u53ef\u4ee5\u53d1\u73b0\u901a\u8fc7ort_session.get_inputs()[0].name\u53ef\u4ee5\u5f97\u5230\u8f93\u5165\u7684tensor\u540d\u79f0\uff0c\u540c\u7406\uff0c\u4e5f\u53ef\u4ee5\u901a\u8fc7ort_session.get_outputs()[0].name\u6765\u83b7\u5f97\u8f93\u51fatensor\u7684\u540d\u79f0\uff0c\u5982\u679c\u4f60\u7684\u7f51\u7edc\u6709\u4e24\u4e2a\u8f93\u51fa\uff0c\u5219\u4f7f\u7528ort_session.get_outputs()[0].name\u548cort_session.get_outputs()[1].name\u6765\u83b7\u5f97\uff0c\u8fd9\u91cc\u5f97\u5230\u7684\u540d\u79f0\u5728\u540e\u7eedtensorrt\u8c03\u7528\u4e2d\u4f1a\u4f7f\u7528<\/code><\/pre>\n\n\n\n
onnx\u8f6cTensorRT\u7684engine\u6587\u4ef6<\/strong>\uff1a<\/p>\n\n\n\n
\nimport tensorrt as trt\nimport sys\nimport os\n\ndef ONNX_build_engine(onnx_file_path, write_engine = False):\n    '''\n    \u901a\u8fc7\u52a0\u8f7donnx\u6587\u4ef6\uff0c\u6784\u5efaengine\n    :param onnx_file_path: onnx\u6587\u4ef6\u8def\u5f84\n    :return: engine\n    '''\n    G_LOGGER = trt.Logger(trt.Logger.WARNING)\n    # 1\u3001\u52a8\u6001\u8f93\u5165\u7b2c\u4e00\u70b9\u5fc5\u987b\u8981\u5199\u7684\n    explicit_batch = 1 << (int)(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH) \n    batch_size = 10 # trt\u63a8\u7406\u65f6\u6700\u5927\u652f\u6301\u7684batchsize\n    with trt.Builder(G_LOGGER) as builder, builder.create_network(explicit_batch) as network, trt.OnnxParser(network, G_LOGGER) as parser:\n        builder.max_batch_size = batch_size\n        config = builder.create_builder_config()\n        config.max_workspace_size = common.GiB(2) #common\u6587\u4ef6\u53ef\u4ee5\u81ea\u5df1\u53bbtensorrt\u5b98\u65b9\u4f8b\u7a0b\u4e0b\u9762\u627e\n        config.set_flag(trt.BuilderFlag.FP16)\n        print('Loading ONNX file from path {}...'.format(onnx_file_path))\n        with open(onnx_file_path, 'rb') as model:\n            print('Beginning ONNX file parsing')\n            parser.parse(model.read())\n        print('Completed parsing of ONNX file')\n        print('Building an engine from file {}; this may take a while...'.format(onnx_file_path))\n        # \u91cd\u70b9 \n        profile = builder.create_optimization_profile() #\u52a8\u6001\u8f93\u5165\u65f6\u5019\u9700\u8981 \u5206\u522b\u4e3a\u6700\u5c0f\u8f93\u5165\u3001\u5e38\u89c4\u8f93\u5165\u3001\u6700\u5927\u8f93\u5165\n        # \u6709\u51e0\u4e2a\u8f93\u5165\u5c31\u8981\u5199\u51e0\u4e2aprofile.set_shape \u540d\u5b57\u548c\u8f6connx\u7684\u65f6\u5019\u8981\u5bf9\u5e94\n        profile.set_shape(\"inputs_1\", (1,3), (1,256), (10,512))\n        profile.set_shape(\"inputs_2\", (1,3), (1,256), (10,512)) \n        profile.set_shape(\"inputs_3\", (1,3), (1,256), (10,512)) \n        profile.set_shape(\"inputs_4\", (1,3), (1,256), (10,512)) \n        config.add_optimization_profile(profile)\n \n        engine = builder.build_engine(network, config)\n        print(\"Completed creating Engine\")\n        # \u4fdd\u5b58engine\u6587\u4ef6\n        if write_engine:\n            engine_file_path = 'correction_fp16.trt'\n            with open(engine_file_path, \"wb\") as f:\n                f.write(engine.serialize())\n        return engine<\/code><\/pre>\n\n\n\n
TensorRT \u52a0\u8f7dengine\u6587\u4ef6\u5e76\u8fdb\u884c\u63a8\u7406<\/strong>\uff1a<\/p>\n\n\n\n
def trt_inference(engine_file):\n    #\u6b64\u5904\u7684\u8f93\u5165\u5e94\u5f53\u8f6c\u6210numpy\u7684array\uff0c\u540c\u65f6dtype\u4e00\u5b9a\u8981\u548c\u539f\u7f51\u7edc\u4e00\u81f4\u4e0d\u7136\u7ed3\u679c\u4f1a\u4e0d\u5bf9\n    inputs_1= np.array(inputs_1, dtype=np.int32, order='C')\n    inputs_2= np.array(inputs_2, dtype=np.int32, order='C')\n    inputs_3= np.array(inputs_3, dtype=np.int32, order='C')\n    inputs_4= np.array(inputs_4, dtype=np.int32, order='C')\n    with get_engine(engine_file) as engine, engine.create_execution_context() as context:\n        #\u589e\u52a0\u90e8\u5206 \u52a8\u6001\u8f93\u5165\u9700\u8981\n        context.active_optimization_profile = 0\n        origin_inputshape=context.get_binding_shape(0)\n        origin_inputshape[0],origin_inputshape[1]=inputs_1.shape\n        context.set_binding_shape(0, (origin_inputshape)) #\u82e5\u6bcf\u4e2a\u8f93\u5165\u7684size\u4e0d\u4e00\u6837\uff0c\u53ef\u6839\u636einputs_i\u7684size\u66f4\u6539\u5bf9\u5e94\u7684context\u4e2d\u7684size\n        context.set_binding_shape(1, (origin_inputshape))\n        context.set_binding_shape(2, (origin_inputshape))\n        context.set_binding_shape(3, (origin_inputshape))\n        #\u589e\u52a0\u4ee3\u7801\u7ed3\u675f\n        inputs, outputs, bindings, stream = common.allocate_buffers(engine, context)\n        # Do inference\n        inputs[0].host = inputs_1\n        inputs[1].host = inputs_2\n        inputs[2].host = inputs_3\n        inputs[3].host = inputs_4\n        trt_outputs = common.do_inference_v2(context, bindings=bindings, inputs=inputs, outputs=outputs, stream=stream)\n\n\ndef get_engine(engine_path ):\n    logger = trt.Logger(trt.Logger.INFO)\n    trt.init_libnvinfer_plugins(logger, namespace=\"\")\n    with open(engine_path, 'rb') as f, trt.Runtime(logger) as runtime:\n        return runtime.deserialize_cuda_engine(f.read())\n<\/code><\/pre>\n\n\n\n
common\u6587\u4ef6\u4e2d\u7684allcate_buffers()\u51fd\u6570\uff1a<\/strong><\/p>\n\n\n\n
def allocate_buffers(engine, context):\n    inputs = []\n    outputs = []\n    bindings = []\n    stream = cuda.Stream()\n    for i, binding in enumerate(engine):\n        size = trt.volume(context.get_binding_shape(i))\n        dtype = trt.nptype(engine.get_binding_dtype(binding))\n        # Allocate host and device buffers\n        host_mem = cuda.pagelocked_empty(size, dtype)\n        device_mem = cuda.mem_alloc(host_mem.nbytes)\n        # Append the device buffer to device bindings.\n        bindings.append(int(device_mem))\n        # Append to the appropriate list.\n        if engine.binding_is_input(binding):\n            inputs.append(HostDeviceMem(host_mem, device_mem))\n        else:\n            outputs.append(HostDeviceMem(host_mem, device_mem))\n    return inputs, outputs, bindings, stream<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
\u6559\u7a0b\u6587\u6863\uff1a \u30101\u3011https:\/\/docs.nvidia.com\/deeplearning\/tensorrt\/ … \u7ee7\u7eed\u9605\u8bfbTensorRT 7 \u52a8\u6001\u8f93\u5165\u548c\u8f93\u51fa<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[14,11],"tags":[],"_links":{"self":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/14469"}],"collection":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/comments?post=14469"}],"version-history":[{"count":38,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/14469\/revisions"}],"predecessor-version":[{"id":14519,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/14469\/revisions\/14519"}],"wp:attachment":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/media?parent=14469"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/categories?post=14469"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/tags?post=14469"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}