#Standard Mode

Please use the hbm_infer.hbm_rpc_session module for Standard Mode .

#API Parameter Descriptions

1. HbmRpcSession Member Method: __init__

   def __init__(
       self,
       host:             str,
       local_hbm_path:   str,
       username:         Optional[str]   = "root",
       remote_root:      Optional[str]   = "/map/hbm_infer/",
       password:         Optional[str]   = "",
       frame_timeout:    Optional[int]   = 90,
       server_timeout:   Optional[int]   = 30,
       with_profile:     Optional[bool]  = False,
       debug:            Optional[bool]  = False,
       compress_option:  Optional[str]   = "NONE",
   ) -> None:

   Initializes an HbmRpcSession object.

   • Parameters

   PARAMETER	DESCRIPTIONS
   host	IP address of the development board.
   local_hbm_path	Local path to the HBM file.
   username	Board-side username.
   remote_root	Root directory for temporary files on the board.
   password	Login password for the development board.
   frame_timeout	Per-frame timeout for gRPC communication in seconds.
   server_timeout	Server timeout in minutes. Server auto-terminates and cleans non-log files after timeout.
   with_profile	Whether to enable time statistics for each stage of inference. The default value is False .
   debug	Enable debug mode retains more logs.
   compress_option	Enable the gRPC compression feature. Optional values are "IN" , "INOUT" , and "NONE" , which indicate enabling compression for request data frames , enabling compression for both request and response data frames , and disabling compression , respectively.

2. HbmRpcSession Member Method: get_input_info

   def get_input_info(self) -> dict:

   Get model input information.

   • Returns

   A dictionary describing the model input information. For specific format details, refer to the example below:

   {
       "input_name0": {
           "valid_shape": [1, 3, 224, 224],
           "tensor_type": "DATA_TYPE_S8",
           "quanti_type": "QUANTI_TYPE_SCALE",
           "quantizeAxis": 0,
           "scale_data": [0.006861070170998573],
           "zero_point_data": [0]
       },
       ...
   }

3. HbmRpcSession Member Method: get_output_info

   def get_output_info(self) -> dict:

   Get model output information.

   • Returns

   Returns a dictionary describing the model output information , with a format consistent with the return value of get_input_info .

4. HbmRpcSession Member Method: show_input_output_info

   def show_input_output_info(self) -> None:

   Print model input and output information.

5. HbmRpcSession Member Method: __call__

   def __call__(self, data) -> (dict | list):

   Perform model inference.

   • Parameters

   PARAMETER	DESCRIPTIONS
   data	Model input data (dictionary type). The value supports two formats: torch.Tensor and numpy.ndarray . Note: The input data must align with the model's input specifications (including name, quantity, shape, and data element type) .

   • Returns

   Model output, of dictionary type. The key is the name of the output tensor, and the value is the output tensor, which has the same type as the model input.

6. HbmRpcSession Member Method: close_server

   def close_server(self) -> None:

   Shut down the server and clean up server-side resources.

7. HbmRpcSession Member Method: get_profile

   def get_profile(self) -> dict:

   To obtain the time statistics for each stage of inference, the with_profile parameter must be set to True , otherwise an error will be raised. The return value is of dictionary type, and its reference format is as follows:

   {
       // Total frame latency (ms)
       "frame_duration": {
           "avg": 6,
           "min": 6,
           "max": 6,
       },
       // Latency from sending a gRPC request to receiving a response (ms)
       "sd2rv_duration": {
           "avg": 5,
           "min": 5,
           "max": 5,
       },
       // Network communication latency (ms)
       "commu_duration": {
           "avg": 4,
           "min": 4,
           "max": 4,
       },
       // Board-side total latency (ms)
       "board_duration": {
           "avg": 1,
           "min": 1,
           "max": 1,
       },
       // Board-side infer latency (ms)
       "infer_duration": {
           "avg": 0.5,
           "min": 0.5,
           "max": 0.5,
       },
       // Board-side preprocess latency (ms)
       "prepr_duration": {
           "avg": 0.3,
           "min": 0.3,
           "max": 0.3,
       },
       // Board-side postprocess latency (ms)
       "pospr_duration": {
           "avg": 0.2,
           "min": 0.2,
           "max": 0.2,
       },
   }

8. HbmRpcSession Member Method: get_profile_last_frame

   def get_profile_last_frame(self) -> dict:

   To obtain the time statistics for each stage of the most recent frame inference, the with_profile parameter must be set to True , otherwise an error will be raised. The return value is of dictionary type, and its reference format is as follows:

   {
       // Total frame latency (ms)
       "frame_duration": 12,
       // Latency from sending a gRPC request to receiving a response (ms)
       "sd2rv_duration": 10,
       // Network communication latency (ms)
       "commu_duration": 6,
       // Board-side total latency (ms)
       "board_duration": 4,
       // Board-side infer latency (ms)
       "infer_duration": 2,
       // Board-side preprocess latency (ms)
       "prepr_duration": 0.5,
       // Board-side postprocess latency (ms)
       "pospr_duration": 0.5,
   }

#Usage Example

import time
import torch
from hbm_infer.hbm_rpc_session import HbmRpcSession

def run_hbm_infer(run_epoch=10):
    # Create session
    sess = HbmRpcSession(
        host=<available_ip>,
        local_hbm_path=<local_hbm_path>
    )
    # Print model input/output information
    sess.show_input_output_info()
    # Prepare input data
    input_data = {
        'img': torch.ones((1, 3, 224, 224), dtype=torch.int8)
    }
    # Execute inference and return results
    for i in range(run_epoch):
        output_data = sess(input_data)
        print([output_data[k].shape for k in output_data])
    # Close server
    sess.close_server()

if __name__ == '__main__':
    run_hbm_infer()