tritonserver学习之九：tritonserver grpc异步模式_在异步协程中调用tritonclient

 void Run(uint16_t port) {
    std::string server_address = absl::StrFormat("0.0.0.0:%d", port);
 
    ServerBuilder builder;
    // Listen on the given address without any authentication mechanism.
    builder.AddListeningPort(server_address, grpc::InsecureServerCredentials());
    // Register "service_" as the instance through which we'll communicate with
    // clients. In this case it corresponds to an *asynchronous* service.
    builder.RegisterService(&service_);
    // Get hold of the completion queue used for the asynchronous communication
    // with the gRPC runtime.
    cq_ = builder.AddCompletionQueue();
    // Finally assemble the server.
    server_ = builder.BuildAndStart();
    std::cout << "Server listening on " << server_address << std::endl;
 
    // Proceed to the server's main loop.
    HandleRpcs();
  }

service_->RequestSayHello(&ctx_, &request_, &responder_, cq_, cq_, this);

/*
 *
 * Copyright 2015 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
 
#include <iostream>
#include <memory>
#include <string>
#include <thread>
 
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "absl/strings/str_format.h"
 
#include <grpc/support/log.h>
#include <grpcpp/grpcpp.h>
 
#ifdef BAZEL_BUILD
#include "examples/protos/helloworld.grpc.pb.h"
#else
#include "helloworld.grpc.pb.h"
#endif
 
ABSL_FLAG(uint16_t, port, 50051, "Server port for the service");
 
using grpc::Server;
using grpc::ServerAsyncResponseWriter;
using grpc::ServerBuilder;
using grpc::ServerCompletionQueue;
using grpc::ServerContext;
using grpc::Status;
using helloworld::Greeter;
using helloworld::HelloReply;
using helloworld::HelloRequest;
 
class ServerImpl final {
 public:
  ~ServerImpl() {
    server_->Shutdown();
    // Always shutdown the completion queue after the server.
    cq_->Shutdown();
  }
 
  // There is no shutdown handling in this code.
  void Run(uint16_t port) {
    std::string server_address = absl::StrFormat("0.0.0.0:%d", port);
 
    ServerBuilder builder;
    // Listen on the given address without any authentication mechanism.
    builder.AddListeningPort(server_address, grpc::InsecureServerCredentials());
    // Register "service_" as the instance through which we'll communicate with
    // clients. In this case it corresponds to an *asynchronous* service.
    builder.RegisterService(&service_);
    // Get hold of the completion queue used for the asynchronous communication
    // with the gRPC runtime.
    cq_ = builder.AddCompletionQueue();
    // Finally assemble the server.
    server_ = builder.BuildAndStart();
    std::cout << "Server listening on " << server_address << std::endl;
 
    // Proceed to the server's main loop.
    HandleRpcs();
  }
 
 private:
  // Class encompasing the state and logic needed to serve a request.
  class CallData {
   public:
    // Take in the "service" instance (in this case representing an asynchronous
    // server) and the completion queue "cq" used for asynchronous communication
    // with the gRPC runtime.
    CallData(Greeter::AsyncService* service, ServerCompletionQueue* cq)
        : service_(service), cq_(cq), responder_(&ctx_), status_(CREATE) {
      // Invoke the serving logic right away.
      Proceed();
    }
 
    void Proceed() {
      if (status_ == CREATE) {
        // Make this instance progress to the PROCESS state.
        status_ = PROCESS;
 
        // As part of the initial CREATE state, we *request* that the system
        // start processing SayHello requests. In this request, "this" acts are
        // the tag uniquely identifying the request (so that different CallData
        // instances can serve different requests concurrently), in this case
        // the memory address of this CallData instance.
        service_->RequestSayHello(&ctx_, &request_, &responder_, cq_, cq_,
                                  this);
      } else if (status_ == PROCESS) {
        // Spawn a new CallData instance to serve new clients while we process
        // the one for this CallData. The instance will deallocate itself as
        // part of its FINISH state.
        new CallData(service_, cq_);
 
        // The actual processing.
        std::string prefix("Hello ");
        reply_.set_message(prefix + request_.name());
 
        // And we are done! Let the gRPC runtime know we've finished, using the
        // memory address of this instance as the uniquely identifying tag for
        // the event.
        status_ = FINISH;
        responder_.Finish(reply_, Status::OK, this);
      } else {
        GPR_ASSERT(status_ == FINISH);
        // Once in the FINISH state, deallocate ourselves (CallData).
        delete this;
      }
    }
 
   private:
    // The means of communication with the gRPC runtime for an asynchronous
    // server.
    Greeter::AsyncService* service_;
    // The producer-consumer queue where for asynchronous server notifications.
    ServerCompletionQueue* cq_;
    // Context for the rpc, allowing to tweak aspects of it such as the use
    // of compression, authentication, as well as to send metadata back to the
    // client.
    ServerContext ctx_;
 
    // What we get from the client.
    HelloRequest request_;
    // What we send back to the client.
    HelloReply reply_;
 
    // The means to get back to the client.
    ServerAsyncResponseWriter<HelloReply> responder_;
 
    // Let's implement a tiny state machine with the following states.
    enum CallStatus { CREATE, PROCESS, FINISH };
    CallStatus status_;  // The current serving state.
  };
 
  // This can be run in multiple threads if needed.
  void HandleRpcs() {
    // Spawn a new CallData instance to serve new clients.
    new CallData(&service_, cq_.get());
    void* tag;  // uniquely identifies a request.
    bool ok;
    while (true) {
      // Block waiting to read the next event from the completion queue. The
      // event is uniquely identified by its tag, which in this case is the
      // memory address of a CallData instance.
      // The return value of Next should always be checked. This return value
      // tells us whether there is any kind of event or cq_ is shutting down.
      GPR_ASSERT(cq_->Next(&tag, &ok));
      GPR_ASSERT(ok);
      static_cast<CallData*>(tag)->Proceed();
    }
  }
 
  std::unique_ptr<ServerCompletionQueue> cq_;
  Greeter::AsyncService service_;
  std::unique_ptr<Server> server_;
};
 
int main(int argc, char** argv) {
  absl::ParseCommandLine(argc, argv);
  ServerImpl server;
  server.Run(absl::GetFlag(FLAGS_port));
 
  return 0;
}

  std::unique_ptr<::grpc::ServerCompletionQueue> common_cq_;  // 普通请求
  std::unique_ptr<::grpc::ServerCompletionQueue> model_infer_cq_;   // 推理请求
  std::unique_ptr<::grpc::ServerCompletionQueue> model_stream_infer_cq_;  // 流式推理请求

TRITONSERVER_Error*
StartGrpcService(
    std::unique_ptr<triton::server::grpc::Server>* service,
    const std::shared_ptr<TRITONSERVER_Server>& server,
    triton::server::TraceManager* trace_manager,
    const std::shared_ptr<triton::server::SharedMemoryManager>& shm_manager)
{
  TRITONSERVER_Error* err = triton::server::grpc::Server::Create(
      server, trace_manager, shm_manager, g_triton_params.grpc_options_,
      service);
  if (err == nullptr) {
    err = (*service)->Start();
  }
 
  if (err != nullptr) {
    service->reset();
  }
 
  return err;
}

void
CommonHandler::Start()
{
  // Use a barrier to make sure we don't return until thread has
  // started.
  auto barrier = std::make_shared<Barrier>(2);
  // 启动一个线程，完成api的注册以及处理
  thread_.reset(new std::thread([this, barrier] {
    // 注册所有函数
    SetUpAllRequests();
    barrier->Wait();
 
    void* tag;
    bool ok;
 
    // 循环等待接收请求
    while (cq_->Next(&tag, &ok)) {
      ICallData* call_data = static_cast<ICallData*>(tag);
      if (!call_data->Process(ok)) {
        LOG_VERBOSE(1) << "Done for " << call_data->Name() << ", "
                       << call_data->Id();
        delete call_data;
      }
    }
  }));
 
  barrier->Wait();
  LOG_VERBOSE(1) << "Thread started for " << Name();
}

void
CommonHandler::RegisterHealthCheck()
{
  auto OnRegisterHealthCheck =
      [this](
          ::grpc::ServerContext* ctx,
          ::grpc::health::v1::HealthCheckRequest* request,
          ::grpc::ServerAsyncResponseWriter<
              ::grpc::health::v1::HealthCheckResponse>* responder,
          void* tag) {
        this->health_service_->RequestCheck(
            ctx, request, responder, this->cq_, this->cq_, tag);
      };
 
  auto OnExecuteHealthCheck = [this](
                                  ::grpc::health::v1::HealthCheckRequest&
                                      request,
                                  ::grpc::health::v1::HealthCheckResponse*
                                      response,
                                  ::grpc::Status* status) {
    bool live = false;
    TRITONSERVER_Error* err =
        TRITONSERVER_ServerIsReady(tritonserver_.get(), &live);
 
    auto serving_status =
        ::grpc::health::v1::HealthCheckResponse_ServingStatus_UNKNOWN;
    if (err == nullptr) {
      serving_status =
          live ? ::grpc::health::v1::HealthCheckResponse_ServingStatus_SERVING
               : ::grpc::health::v1::
                     HealthCheckResponse_ServingStatus_NOT_SERVING;
    }
    response->set_status(serving_status);
 
    GrpcStatusUtil::Create(status, err);
    TRITONSERVER_ErrorDelete(err);
  };
 
  const std::pair<std::string, std::string>& restricted_kv =
      restricted_keys_.Get(RestrictedCategory::HEALTH);
  new CommonCallData<
      ::grpc::ServerAsyncResponseWriter<
          ::grpc::health::v1::HealthCheckResponse>,
      ::grpc::health::v1::HealthCheckRequest,
      ::grpc::health::v1::HealthCheckResponse>(
      "Check", 0, OnRegisterHealthCheck, OnExecuteHealthCheck,
      false /* async */, cq_, restricted_kv, response_delay_);
}

template <typename ResponderType, typename RequestType, typename ResponseType>
bool
CommonCallData<ResponderType, RequestType, ResponseType>::Process(bool rpc_ok)
{
  LOG_VERBOSE(1) << "Process for " << name_ << ", rpc_ok=" << rpc_ok << ", "
                 << id_ << " step " << step_;
 
  // If RPC failed on a new request then the server is shutting down
  // and so we should do nothing (including not registering for a new
  // request). If RPC failed on a non-START step then there is nothing
  // we can do since we one execute one step.
  const bool shutdown = (!rpc_ok && (step_ == Steps::START));
  if (shutdown) {
    if (async_thread_.joinable()) {
      async_thread_.join();
    }
    step_ = Steps::FINISH;
  }
 
  if (step_ == Steps::START) {
    // Start a new request to replace this one...
    if (!shutdown) {
      new CommonCallData<ResponderType, RequestType, ResponseType>(
          name_, id_ + 1, OnRegister_, OnExecute_, async_, cq_, restricted_kv_,
          response_delay_);
    }
 
    if (!async_) {
      // For synchronous calls, execute and write response
      // here.
      Execute();
      WriteResponse();
    } else {
      // For asynchronous calls, delegate the execution to another
      // thread.
      step_ = Steps::ISSUED;
      async_thread_ = std::thread(&CommonCallData::Execute, this);
    }
  } else if (step_ == Steps::WRITEREADY) {
    // Will only come here for asynchronous mode.
    WriteResponse();
  } else if (step_ == Steps::COMPLETE) {
    step_ = Steps::FINISH;
  }
 
  return step_ != Steps::FINISH;
}