From 8f579f27dbaf3603b166e07c08b5309d436d77eb Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Mon, 24 Jan 2022 15:11:08 +0800 Subject: [PATCH 01/46] Add some debug info in security_barrier_camera_demo. Signed-off-by: Wang, Yang --- .../utils/include/utils/threads_common.hpp | 3 ++ .../python/face_recognition_demo.py | 2 +- .../security_barrier_camera_demo/cpp/main.cpp | 44 ++++++++++++++++++- .../cpp/net_wrappers.hpp | 28 ++++++++++-- 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/demos/common/cpp/utils/include/utils/threads_common.hpp b/demos/common/cpp/utils/include/utils/threads_common.hpp index f0e5cbf3dec..7fc0c2130b1 100644 --- a/demos/common/cpp/utils/include/utils/threads_common.hpp +++ b/demos/common/cpp/utils/include/utils/threads_common.hpp @@ -66,8 +66,10 @@ class Worker { } void runThreads() { running = true; + slog::debug << "thread pool size: " << threadPool.size() << slog::endl; for (std::thread& t : threadPool) { t = std::thread(&Worker::threadFunc, this); + slog::debug << "thread is pushing now." << slog::endl; } } void push(std::shared_ptr task) { @@ -77,6 +79,7 @@ class Worker { tasksCondVar.notify_one(); } void threadFunc() { + slog::debug << "thread is running now." << slog::endl; while (running) { std::unique_lock lk(tasksMutex); while (running && tasks.empty()) { diff --git a/demos/face_recognition_demo/python/face_recognition_demo.py b/demos/face_recognition_demo/python/face_recognition_demo.py index be777c7edbb..1b48ed2687d 100755 --- a/demos/face_recognition_demo/python/face_recognition_demo.py +++ b/demos/face_recognition_demo/python/face_recognition_demo.py @@ -42,7 +42,7 @@ log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.DEBUG, stream=sys.stdout) -DEVICE_KINDS = ['CPU', 'GPU', 'MYRIAD', 'HETERO', 'HDDL'] +DEVICE_KINDS = ['CPU', 'GPU', 'AUTO', 'AUTO:CPU,GPU', 'MULTI:CPU,GPU', 'MYRIAD', 'HETERO', 'HDDL'] def build_argparser(): diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index c19087f0895..3bd5a97081a 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -220,7 +220,7 @@ class ClassifiersAggregator { std::mutex& printMutex = static_cast(sharedVideoFrame.get())->context.classifiersAggregatorPrintMutex; printMutex.lock(); if (FLAGS_r && !rawDetections.empty()) { - slog::debug << "Frame #: " << sharedVideoFrame->frameId << slog::endl; + //slog::debug << "--Channel #" << sharedVideoFrame->sourceID <<"--Frame #" << sharedVideoFrame->frameId << "-----" << slog::endl; slog::debug << rawDetections; // destructor assures that none uses the container for (const std::string& rawAttribute : rawAttributes.container) { @@ -333,6 +333,7 @@ bool Drawer::isReady() { void Drawer::process() { const int64_t frameId = sharedVideoFrame->frameId; + //slog::debug <<"Drawer::process for channel-frameid [" << sharedVideoFrame->sourceID <<" - " << sharedVideoFrame->frameId << "]....." << slog::endl; Context& context = static_cast(sharedVideoFrame.get())->context; std::map& gridMats = context.drawersContext.gridMats; context.drawersContext.drawerMutex.lock(); @@ -392,6 +393,7 @@ void Drawer::process() { } void ResAggregator::process() { + //slog::debug <<"ResAggregator::process for channel-frameid [" << sharedVideoFrame->sourceID <<" - " << sharedVideoFrame->frameId << "]....." << slog::endl; Context& context = static_cast(sharedVideoFrame.get())->context; context.freeDetectionInfersCount += context.detectorsInfers.inferRequests.lockedSize(); context.frameCounter++; @@ -428,7 +430,7 @@ bool DetectionsProcessor::isReady() { if (requireGettingNumberOfDetections) { classifiersAggregator = std::make_shared(sharedVideoFrame); std::list results; - results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections); + results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->sourceID, sharedVideoFrame->frameId, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections); for (Detector::Result result : results) { switch (result.label) { case 1: @@ -481,6 +483,7 @@ bool DetectionsProcessor::isReady() { } void DetectionsProcessor::process() { + //slog::debug <<"DetectionProcessor::process for channel-frameid [" << sharedVideoFrame->sourceID <<" - " << sharedVideoFrame->frameId << "]....." << slog::endl; Context& context = static_cast(sharedVideoFrame.get())->context; if (!FLAGS_m_va.empty()) { auto vehicleRectsIt = vehicleRects.begin(); @@ -586,6 +589,20 @@ void InferTask::process() { detectorsInfers.inferRequests.container.pop_back(); detectorsInfers.inferRequests.mutex.unlock(); + //for (int i = 0; i < sharedVideoFrame->frame.rows; i++) { + // if(i == sharedVideoFrame->frame.rows - 1) + // slog::debug <<"InferTask::process for channel-frameid-imgsize [" << sharedVideoFrame->sourceID <<" - " << sharedVideoFrame->frameId << "-" << sharedVideoFrame->frame.total() << "]....." << slog::endl; + // for (int j = 0; j < sharedVideoFrame->frame.cols; j++) { + // auto bgr = sharedVideoFrame->frame.at(i,j); + // int a = bgr[0]; + // int b = bgr[1]; + // int c = bgr[2]; + // if(i == sharedVideoFrame->frame.rows - 1) + // std::cout << a <<" " << b << " " << c << " "; + // } + // if(i == sharedVideoFrame->frame.rows - 1) + // std::cout << std::endl; + //} context.inferTasksContext.detector.setImage(inferRequest, sharedVideoFrame->frame); inferRequest.get().set_callback( @@ -607,6 +624,7 @@ void InferTask::process() { bool Reader::isReady() { Context& context = static_cast(sharedVideoFrame.get())->context; context.readersContext.lastCapturedFrameIdsMutexes[sharedVideoFrame->sourceID].lock(); + // Look for the next frame if (context.readersContext.lastCapturedFrameIds[sharedVideoFrame->sourceID] + 1 == sharedVideoFrame->frameId) { return true; } else { @@ -616,6 +634,7 @@ bool Reader::isReady() { } void Reader::process() { + //slog::debug <<"Reader::process for channel-frameid [" << sharedVideoFrame->sourceID <<" - " << sharedVideoFrame->frameId << "]....." << slog::endl; unsigned sourceID = sharedVideoFrame->sourceID; sharedVideoFrame->timestamp = std::chrono::steady_clock::now(); Context& context = static_cast(sharedVideoFrame.get())->context; @@ -725,6 +744,7 @@ int main(int argc, char* argv[]) { device_nstreams["CPU"] = core.get_property("CPU", ov::streams::num); } +<<<<<<< HEAD if ("GPU" == device) { core.set_property("GPU", ov::streams::num(device_nstreams.count("GPU") > 0 ? ov::streams::Num(device_nstreams["GPU"]) : ov::streams::AUTO)); @@ -735,6 +755,23 @@ int main(int argc, char* argv[]) { core.set_property("GPU", ov::intel_gpu::hint::queue_throttle(ov::intel_gpu::hint::ThrottleLevel(1))); } } +======= + //if ("GPU" == device) { + // // Load any user-specified clDNN Extensions + // if (!FLAGS_c.empty()) { + // ie.SetConfig({ { InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE, FLAGS_c } }, "GPU"); + // } + // ie.SetConfig({{ CONFIG_KEY(GPU_THROUGHPUT_STREAMS), + // (device_nstreams.count("GPU") > 0 ? std::to_string(device_nstreams.at("GPU")) : + // CONFIG_VALUE(GPU_THROUGHPUT_AUTO)) }}, "GPU"); + // device_nstreams["GPU"] = std::stoi(ie.GetConfig("GPU", CONFIG_KEY(GPU_THROUGHPUT_STREAMS)).as()); + // if (devices.end() != devices.find("CPU")) { + // // multi-device execution with the CPU + GPU performs best with GPU trottling hint, + // // which releases another CPU thread (that is otherwise used by the GPU driver for active polling) + // ie.SetConfig({{ GPU_CONFIG_KEY(PLUGIN_THROTTLE), "1" }}, "GPU"); + // } + //} +>>>>>>> Add some debug info in security_barrier_camera_demo. } // Graph tagging via config options @@ -806,12 +843,15 @@ int main(int argc, char* argv[]) { context.drawersContext.drawersWorker = worker; context.resAggregatorsWorker = worker; + int readerCount = 0; for (uint64_t i = 0; i < FLAGS_n_iqs; i++) { for (unsigned sourceID = 0; sourceID < inputChannels.size(); sourceID++) { VideoFrame::Ptr sharedVideoFrame = std::make_shared(context, sourceID, i); worker->push(std::make_shared(sharedVideoFrame)); + readerCount++; } } + slog::debug << "Reader Count: " << readerCount << slog::endl; // Running worker->runThreads(); diff --git a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp index 00519a761a9..e36e2468a16 100644 --- a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp +++ b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp @@ -119,14 +119,32 @@ class Detector { // resize and copy data from image to tensor using OpenCV resize2tensor(img, inputTensor); } + /* + InferenceEngine::SizeVector blobSize = input->getTensorDesc().getDims(); + const size_t width = blobSize[3]; + const size_t height = blobSize[2]; + const size_t channels = blobSize[1]; + InferenceEngine::LockedMemory blobMapped = InferenceEngine::as(input)->wmap(); + uint8_t* blobData = blobMapped.as(); + for (size_t c = 0; c < channels; c++) + for (size_t h = 0; h < height; h++) + for (size_t w = 0; w < width; w++) + std::cout << blobData[ c * width * height + h * width + w]; + std::cout << std::endl; + */ } - std::list getResults(ov::InferRequest& inferRequest, cv::Size upscale, std::vector& rawResults) { - // there is no big difference if InferReq of detector from another device is passed - // because the processing is the same for the same topology + std::list getResults(ov::InferRequest& inferRequest, const int64_t channelID, const int64_t frameID, cv::Size upscale, std::vector& rawResults) { + // there is no big difference if InferReq of detector from another device is passed because the processing is the same for the same topology std::list results; ov::Tensor output_tensor = inferRequest.get_tensor(m_detectorOutputName); const float* const detections = output_tensor.data(); + slog::debug << "------- Channel #" << channelID << " - Frame #" << frameID << " ----- blob size #---"<< blob->size() << slog::endl; + for (size_t i = 0; i < blob->size(); i++) + { + std::cout << detections[i] << " "; + } + std::cout << std::endl; // pretty much regular SSD post-processing for (int i = 0; i < maxProposalCount; i++) { float image_id = detections[i * objectSize + 0]; // in case of batch @@ -148,8 +166,12 @@ class Detector { std::ostringstream rawResultsStream; rawResultsStream << "[" << i << "," << label << "] element, prob = " << confidence << " (" << rect.x << "," << rect.y << ")-(" << rect.width << "," << rect.height << ")"; + rawResults.push_back(rawResultsStream.str()); } + + if(results.size() != 3) + slog::debug << "==== Channel #" << channelID << "-Frame #" << frameID << "-ROI Number #" << results.size() << " ====" << slog::endl; return results; } From fef9a0a9e5da60083a774a6859739523b50d04cf Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Mon, 7 Mar 2022 09:38:24 +0800 Subject: [PATCH 02/46] Update. Signed-off-by: Wang, Yang --- .../security_barrier_camera_demo/cpp/main.cpp | 18 ------------------ .../cpp/net_wrappers.hpp | 7 +------ 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index 3bd5a97081a..c905397a47b 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -744,7 +744,6 @@ int main(int argc, char* argv[]) { device_nstreams["CPU"] = core.get_property("CPU", ov::streams::num); } -<<<<<<< HEAD if ("GPU" == device) { core.set_property("GPU", ov::streams::num(device_nstreams.count("GPU") > 0 ? ov::streams::Num(device_nstreams["GPU"]) : ov::streams::AUTO)); @@ -755,23 +754,6 @@ int main(int argc, char* argv[]) { core.set_property("GPU", ov::intel_gpu::hint::queue_throttle(ov::intel_gpu::hint::ThrottleLevel(1))); } } -======= - //if ("GPU" == device) { - // // Load any user-specified clDNN Extensions - // if (!FLAGS_c.empty()) { - // ie.SetConfig({ { InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE, FLAGS_c } }, "GPU"); - // } - // ie.SetConfig({{ CONFIG_KEY(GPU_THROUGHPUT_STREAMS), - // (device_nstreams.count("GPU") > 0 ? std::to_string(device_nstreams.at("GPU")) : - // CONFIG_VALUE(GPU_THROUGHPUT_AUTO)) }}, "GPU"); - // device_nstreams["GPU"] = std::stoi(ie.GetConfig("GPU", CONFIG_KEY(GPU_THROUGHPUT_STREAMS)).as()); - // if (devices.end() != devices.find("CPU")) { - // // multi-device execution with the CPU + GPU performs best with GPU trottling hint, - // // which releases another CPU thread (that is otherwise used by the GPU driver for active polling) - // ie.SetConfig({{ GPU_CONFIG_KEY(PLUGIN_THROTTLE), "1" }}, "GPU"); - // } - //} ->>>>>>> Add some debug info in security_barrier_camera_demo. } // Graph tagging via config options diff --git a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp index e36e2468a16..13961a2e075 100644 --- a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp +++ b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp @@ -139,12 +139,7 @@ class Detector { std::list results; ov::Tensor output_tensor = inferRequest.get_tensor(m_detectorOutputName); const float* const detections = output_tensor.data(); - slog::debug << "------- Channel #" << channelID << " - Frame #" << frameID << " ----- blob size #---"<< blob->size() << slog::endl; - for (size_t i = 0; i < blob->size(); i++) - { - std::cout << detections[i] << " "; - } - std::cout << std::endl; + slog::debug << "------- Channel #" << channelID << " - Frame #" << frameID << slog::endl; // pretty much regular SSD post-processing for (int i = 0; i < maxProposalCount; i++) { float image_id = detections[i * objectSize + 0]; // in case of batch From d101b6778182f9ccd8c300e9f3510337c38cf3c3 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Thu, 10 Mar 2022 15:10:15 +0800 Subject: [PATCH 03/46] Update the debug msg and Add result paser for security_barrier_camera_demo. Signed-off-by: Wang, Yang --- .../utils/include/utils/threads_common.hpp | 3 -- .../security_barrier_camera_demo/cpp/main.cpp | 6 +-- .../cpp/net_wrappers.hpp | 10 ++-- demos/tests/cases.py | 50 ++++++++++++++++--- demos/tests/run_tests.py | 5 +- 5 files changed, 51 insertions(+), 23 deletions(-) diff --git a/demos/common/cpp/utils/include/utils/threads_common.hpp b/demos/common/cpp/utils/include/utils/threads_common.hpp index 7fc0c2130b1..f0e5cbf3dec 100644 --- a/demos/common/cpp/utils/include/utils/threads_common.hpp +++ b/demos/common/cpp/utils/include/utils/threads_common.hpp @@ -66,10 +66,8 @@ class Worker { } void runThreads() { running = true; - slog::debug << "thread pool size: " << threadPool.size() << slog::endl; for (std::thread& t : threadPool) { t = std::thread(&Worker::threadFunc, this); - slog::debug << "thread is pushing now." << slog::endl; } } void push(std::shared_ptr task) { @@ -79,7 +77,6 @@ class Worker { tasksCondVar.notify_one(); } void threadFunc() { - slog::debug << "thread is running now." << slog::endl; while (running) { std::unique_lock lk(tasksMutex); while (running && tasks.empty()) { diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index c905397a47b..a05025722ac 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -220,8 +220,8 @@ class ClassifiersAggregator { std::mutex& printMutex = static_cast(sharedVideoFrame.get())->context.classifiersAggregatorPrintMutex; printMutex.lock(); if (FLAGS_r && !rawDetections.empty()) { - //slog::debug << "--Channel #" << sharedVideoFrame->sourceID <<"--Frame #" << sharedVideoFrame->frameId << "-----" << slog::endl; - slog::debug << rawDetections; + for (const std::string& rawDetection : rawDetections) + slog::debug << rawDetection << slog::endl; // destructor assures that none uses the container for (const std::string& rawAttribute : rawAttributes.container) { slog::debug << rawAttribute << slog::endl; @@ -431,6 +431,7 @@ bool DetectionsProcessor::isReady() { classifiersAggregator = std::make_shared(sharedVideoFrame); std::list results; results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->sourceID, sharedVideoFrame->frameId, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections); + for (Detector::Result result : results) { switch (result.label) { case 1: @@ -833,7 +834,6 @@ int main(int argc, char* argv[]) { readerCount++; } } - slog::debug << "Reader Count: " << readerCount << slog::endl; // Running worker->runThreads(); diff --git a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp index 13961a2e075..a52370080db 100644 --- a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp +++ b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp @@ -139,7 +139,6 @@ class Detector { std::list results; ov::Tensor output_tensor = inferRequest.get_tensor(m_detectorOutputName); const float* const detections = output_tensor.data(); - slog::debug << "------- Channel #" << channelID << " - Frame #" << frameID << slog::endl; // pretty much regular SSD post-processing for (int i = 0; i < maxProposalCount; i++) { float image_id = detections[i * objectSize + 0]; // in case of batch @@ -159,14 +158,11 @@ class Detector { rect.height = static_cast(detections[i * objectSize + 6] * upscale.height) - rect.y; results.push_back(Result{label, confidence, rect}); std::ostringstream rawResultsStream; - rawResultsStream << "[" << i << "," << label << "] element, prob = " << confidence - << " (" << rect.x << "," << rect.y << ")-(" << rect.width << "," << rect.height << ")"; - + rawResultsStream << "ChannelId:" << channelID << ",FrameId:" << frameID << ","; + rawResultsStream << "ObjId:" << i << "," << label << "," << confidence + << "," << rect.x << "," << rect.y << "," << rect.width << "," << rect.height; rawResults.push_back(rawResultsStream.str()); } - - if(results.size() != 3) - slog::debug << "==== Channel #" << channelID << "-Frame #" << frameID << "-ROI Number #" << results.size() << " ====" << slog::endl; return results; } diff --git a/demos/tests/cases.py b/demos/tests/cases.py index 8aa958849c1..8dec28bc973 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -27,11 +27,10 @@ # TODO with Python3.7 use namedtuple defaults instead TestCase.__new__.__defaults__ = [], - class Demo: IMPLEMENTATION_TYPES = set() - def __init__(self, name, implementation, model_keys=None, device_keys=None, test_cases=None): + def __init__(self, name, implementation, model_keys=None, device_keys=None, test_cases=None, parser=None): self.implementation = implementation self.subdirectory = name + '/' + implementation self.device_keys = device_keys @@ -40,7 +39,8 @@ def __init__(self, name, implementation, model_keys=None, device_keys=None, test self.test_cases = test_cases self._exec_name = self.subdirectory.replace('/', '_') - self.parser = None + self.parser = parser + self.results = {} Demo.IMPLEMENTATION_TYPES.add(implementation) @@ -69,9 +69,13 @@ def add_parser(self, parser): self.parser = parser(self) return self - def parse_output(self, output, test_case, device): + def parse_output(self, output, device, test_case_index): if self.parser: - self.parser(output, test_case, device) + tmp = self.parser(output) + if device not in self.results: + self.results[device] = {} + self.results[device][test_case_index] = tmp + print (self.results) def update_option(self, updated_options): for case in self.test_cases[:]: @@ -122,8 +126,8 @@ def set_precisions(self, precisions, model_info): class CppDemo(Demo): - def __init__(self, name, implementation='cpp', model_keys=None, device_keys=None, test_cases=None): - super().__init__(name, implementation, model_keys, device_keys, test_cases) + def __init__(self, name, implementation='cpp', model_keys=None, device_keys=None, test_cases=None, parser=None): + super().__init__(name, implementation, model_keys, device_keys, test_cases, parser) self._exec_name = self._exec_name.replace('_cpp', '') @@ -151,7 +155,6 @@ def join_cases(*args): for case in args: extra_models.update(case.extra_models) return TestCase(options=options, extra_models=list(case.extra_models)) - def combine_cases(*args): return [join_cases(*combination) for combination in itertools.product(*[[arg] if isinstance(arg, TestCase) else arg for arg in args])] @@ -160,6 +163,35 @@ def combine_cases(*args): def single_option_cases(key, *args): return [TestCase(options={} if arg is None else {key: arg}) for arg in args] +def security_barrier_camera_damo_parser(output): + output = output.split('\n') + channel_results = {} + for line in output: + if "ChannelID" in line: + print (line) + if "DEBUG" not in line: + continue + item = line[line.find('Channel') : ].split(',') + # Channel ID + frame_results = {} + channel = item[0].split(':') + if channel[1] not in channel_results: + channel_results[channel[1]] = frame_results + + # Frame ID + object_results = {} + frame = item[1].split(':') + if frame[1] not in channel_results[channel[1]]: + channel_results[channel[1]][frame[1]] = object_results + + # Object ID + label_prob_pos_results = [] + objid = item[2].split(':') + if objid[1] not in channel_results[channel[1]][frame[1]]: + channel_results[channel[1]][frame[1]][objid[1]] = label_prob_pos_results + channel_results[channel[1]][frame[1]][objid[1]] = item[3:] + + return channel_results DEMOS = [ @@ -590,9 +622,11 @@ def single_option_cases(key, *args): CppDemo(name='security_barrier_camera_demo', model_keys=['-m', '-m_lpr', '-m_va'], device_keys=['-d', '-d_lpr', '-d_va'], + parser=security_barrier_camera_damo_parser, test_cases=combine_cases( TestCase(options={'-no_show': None, **MONITORS, + '-r': None, '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}), TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0106')}), single_option_cases('-m_lpr', diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index 2616c26fea8..816f20b7c96 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -33,6 +33,7 @@ import contextlib import csv import json +from operator import contains import os import platform import shlex @@ -321,13 +322,13 @@ def option_to_args(key, value): stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf-8', env=demo_environment) execution_time = timeit.default_timer() - start_time - demo.parse_output(output, test_case, device) + demo.parse_output(output, device, test_case_index) except subprocess.CalledProcessError as e: print(e.output) print('Exit code:', e.returncode) num_failures += 1 execution_time = -1 - + if args.report_file: collect_result(demo.subdirectory, device, case_model_names, execution_time, args.report_file) From 7d5ff2694037de4873021b0f5dc2de724e1de2d4 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Thu, 10 Mar 2022 16:11:21 +0800 Subject: [PATCH 04/46] Add correctness checker interface for demo. Signed-off-by: Wang, Yang --- demos/tests/cases.py | 12 +++++++++++- demos/tests/run_tests.py | 7 +++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/demos/tests/cases.py b/demos/tests/cases.py index 8dec28bc973..18ef0df977e 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -30,7 +30,7 @@ class Demo: IMPLEMENTATION_TYPES = set() - def __init__(self, name, implementation, model_keys=None, device_keys=None, test_cases=None, parser=None): + def __init__(self, name, implementation, model_keys=None, device_keys=None, test_cases=None, parser=None, correctness_checker=None): self.implementation = implementation self.subdirectory = name + '/' + implementation self.device_keys = device_keys @@ -40,6 +40,7 @@ def __init__(self, name, implementation, model_keys=None, device_keys=None, test self._exec_name = self.subdirectory.replace('/', '_') self.parser = parser + self.correctness_checker = correctness_checker self.results = {} Demo.IMPLEMENTATION_TYPES.add(implementation) @@ -77,6 +78,15 @@ def parse_output(self, output, device, test_case_index): self.results[device][test_case_index] = tmp print (self.results) + def add_correctness_check(self, correctness_check): + self.correctness_checker = correctness_check + return self + + def results_correctness_check(self): + if not self.correctness_checker: + return True + return self.correctness_checker() + def update_option(self, updated_options): for case in self.test_cases[:]: self.update_case(case, updated_options, with_replacement=True) diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index 816f20b7c96..7c82d6bc0b7 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -331,6 +331,13 @@ def option_to_args(key, value): if args.report_file: collect_result(demo.subdirectory, device, case_model_names, execution_time, args.report_file) + if not demo.results_correctness_check(): + print ('Results Correct check failed for Demo: ', demo.subdirectory) + else: + print ('Results Correct check Passed') + + + print() From 3551ada7db42325617f817d159060cba9a088d7a Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Mon, 14 Mar 2022 09:02:04 +0800 Subject: [PATCH 05/46] Implement parser and correcteness checker for security_barrier_camera_damo. Signed-off-by: Wang, Yang --- demos/tests/cases.py | 23 ++++++++++++++++++++--- demos/tests/run_tests.py | 4 ++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/demos/tests/cases.py b/demos/tests/cases.py index 18ef0df977e..e6c793fbb06 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -85,7 +85,7 @@ def add_correctness_check(self, correctness_check): def results_correctness_check(self): if not self.correctness_checker: return True - return self.correctness_checker() + return self.correctness_checker(self.results) def update_option(self, updated_options): for case in self.test_cases[:]: @@ -136,8 +136,8 @@ def set_precisions(self, precisions, model_info): class CppDemo(Demo): - def __init__(self, name, implementation='cpp', model_keys=None, device_keys=None, test_cases=None, parser=None): - super().__init__(name, implementation, model_keys, device_keys, test_cases, parser) + def __init__(self, name, implementation='cpp', model_keys=None, device_keys=None, test_cases=None, parser=None, correctness_checker=None): + super().__init__(name, implementation, model_keys, device_keys, test_cases, parser, correctness_checker) self._exec_name = self._exec_name.replace('_cpp', '') @@ -173,6 +173,22 @@ def combine_cases(*args): def single_option_cases(key, *args): return [TestCase(options={} if arg is None else {key: arg}) for arg in args] +def security_barrier_camera_damo_correctness_checker(results): + flag = False + if "CPU" in results and "AUTO:CPU" in results: + if results['CPU'] == results['AUTO:CPU']: + flag = True + else: + print ("CPU vs AUTO:CPU have inconsistent results") + + if "GPU" in results and "AUTO:GPU" in results: + if results['GPU'] == results['AUTO:GPU']: + flag = True + else: + print ("GPU vs AUTO:GPU have inconsistent results") + + return flag + def security_barrier_camera_damo_parser(output): output = output.split('\n') channel_results = {} @@ -632,6 +648,7 @@ def security_barrier_camera_damo_parser(output): CppDemo(name='security_barrier_camera_demo', model_keys=['-m', '-m_lpr', '-m_va'], device_keys=['-d', '-d_lpr', '-d_va'], + correctness_checker=security_barrier_camera_damo_correctness_checker, parser=security_barrier_camera_damo_parser, test_cases=combine_cases( TestCase(options={'-no_show': None, diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index 7c82d6bc0b7..f1da11f8bd1 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -332,9 +332,9 @@ def option_to_args(key, value): if args.report_file: collect_result(demo.subdirectory, device, case_model_names, execution_time, args.report_file) if not demo.results_correctness_check(): - print ('Results Correct check failed for Demo: ', demo.subdirectory) + print ('Results Correctness checking failed for Demo: ', demo.subdirectory) else: - print ('Results Correct check Passed') + print ('Results Correctness checking Passed') From 3fb15e00f3611e93ef33c9885a3d0d4493968513 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 16 Mar 2022 14:42:43 +0800 Subject: [PATCH 06/46] Add correctness checker script and instantiate checker of demo security_barrier_camera_dem. Signed-off-by: Wang, Yang --- demos/tests/cases.py | 79 ++--------------- demos/tests/correctness_checker.py | 134 +++++++++++++++++++++++++++++ demos/tests/run_tests.py | 29 ++++--- 3 files changed, 162 insertions(+), 80 deletions(-) create mode 100755 demos/tests/correctness_checker.py diff --git a/demos/tests/cases.py b/demos/tests/cases.py index e6c793fbb06..860e7dedebc 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -27,10 +27,11 @@ # TODO with Python3.7 use namedtuple defaults instead TestCase.__new__.__defaults__ = [], + class Demo: IMPLEMENTATION_TYPES = set() - def __init__(self, name, implementation, model_keys=None, device_keys=None, test_cases=None, parser=None, correctness_checker=None): + def __init__(self, name, implementation, model_keys=None, device_keys=None, test_cases=None): self.implementation = implementation self.subdirectory = name + '/' + implementation self.device_keys = device_keys @@ -39,9 +40,7 @@ def __init__(self, name, implementation, model_keys=None, device_keys=None, test self.test_cases = test_cases self._exec_name = self.subdirectory.replace('/', '_') - self.parser = parser - self.correctness_checker = correctness_checker - self.results = {} + self.parser = None Demo.IMPLEMENTATION_TYPES.add(implementation) @@ -70,22 +69,9 @@ def add_parser(self, parser): self.parser = parser(self) return self - def parse_output(self, output, device, test_case_index): + def parse_output(self, output, test_case, device): if self.parser: - tmp = self.parser(output) - if device not in self.results: - self.results[device] = {} - self.results[device][test_case_index] = tmp - print (self.results) - - def add_correctness_check(self, correctness_check): - self.correctness_checker = correctness_check - return self - - def results_correctness_check(self): - if not self.correctness_checker: - return True - return self.correctness_checker(self.results) + self.parser(output, test_case, device) def update_option(self, updated_options): for case in self.test_cases[:]: @@ -136,8 +122,8 @@ def set_precisions(self, precisions, model_info): class CppDemo(Demo): - def __init__(self, name, implementation='cpp', model_keys=None, device_keys=None, test_cases=None, parser=None, correctness_checker=None): - super().__init__(name, implementation, model_keys, device_keys, test_cases, parser, correctness_checker) + def __init__(self, name, implementation='cpp', model_keys=None, device_keys=None, test_cases=None): + super().__init__(name, implementation, model_keys, device_keys, test_cases) self._exec_name = self._exec_name.replace('_cpp', '') @@ -165,6 +151,7 @@ def join_cases(*args): for case in args: extra_models.update(case.extra_models) return TestCase(options=options, extra_models=list(case.extra_models)) + def combine_cases(*args): return [join_cases(*combination) for combination in itertools.product(*[[arg] if isinstance(arg, TestCase) else arg for arg in args])] @@ -173,51 +160,6 @@ def combine_cases(*args): def single_option_cases(key, *args): return [TestCase(options={} if arg is None else {key: arg}) for arg in args] -def security_barrier_camera_damo_correctness_checker(results): - flag = False - if "CPU" in results and "AUTO:CPU" in results: - if results['CPU'] == results['AUTO:CPU']: - flag = True - else: - print ("CPU vs AUTO:CPU have inconsistent results") - - if "GPU" in results and "AUTO:GPU" in results: - if results['GPU'] == results['AUTO:GPU']: - flag = True - else: - print ("GPU vs AUTO:GPU have inconsistent results") - - return flag - -def security_barrier_camera_damo_parser(output): - output = output.split('\n') - channel_results = {} - for line in output: - if "ChannelID" in line: - print (line) - if "DEBUG" not in line: - continue - item = line[line.find('Channel') : ].split(',') - # Channel ID - frame_results = {} - channel = item[0].split(':') - if channel[1] not in channel_results: - channel_results[channel[1]] = frame_results - - # Frame ID - object_results = {} - frame = item[1].split(':') - if frame[1] not in channel_results[channel[1]]: - channel_results[channel[1]][frame[1]] = object_results - - # Object ID - label_prob_pos_results = [] - objid = item[2].split(':') - if objid[1] not in channel_results[channel[1]][frame[1]]: - channel_results[channel[1]][frame[1]][objid[1]] = label_prob_pos_results - channel_results[channel[1]][frame[1]][objid[1]] = item[3:] - - return channel_results DEMOS = [ @@ -648,12 +590,9 @@ def security_barrier_camera_damo_parser(output): CppDemo(name='security_barrier_camera_demo', model_keys=['-m', '-m_lpr', '-m_va'], device_keys=['-d', '-d_lpr', '-d_va'], - correctness_checker=security_barrier_camera_damo_correctness_checker, - parser=security_barrier_camera_damo_parser, test_cases=combine_cases( - TestCase(options={'-no_show': None, + TestCase(options={'-no_show': None, '-r' : None, **MONITORS, - '-r': None, '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}), TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0106')}), single_option_cases('-m_lpr', diff --git a/demos/tests/correctness_checker.py b/demos/tests/correctness_checker.py new file mode 100755 index 00000000000..590b23397ea --- /dev/null +++ b/demos/tests/correctness_checker.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from unittest import case + +class Demo(ABC): + def __init__(self, name, implementation): + self.name = name + self.implementation = implementation + self.subdirectory = name + '/' + implementation + self.results = {} + + @abstractmethod + def parser(self): + pass + + @abstractmethod + def checker(self): + pass + +class CPPDemo(Demo): + def __init__(self, name, implementation='cpp'): + super().__init__(name, implementation) + self.results = {} + pass + + def parser(self): + fo = None + try: + fo = open('/tmp/' + self.subdirectory + '/results.log', 'r') + except IOError as err: + print("File error: " + str(err)) + output = [i.rstrip() for i in fo.readlines()] + device = '' + case_index = '' + index = 1 + while index < len(output): + if "Device" in output[index]: + device = output[index][output[index].find(":") + 1:] + if device not in self.results: + self.results[device] = {} + + if "CaseId" in output[index]: + case_index = output[index][output[index].find(":") + 1:] + if case_index not in self.results[device]: + self.results[device][case_index] = {} + + if "Execution_time" in output[index]: + execution_time = output[index].split(':')[1] + if execution_time == '-1': + while index < len(output) and 'Device' not in output[index]: + index += 1 + continue + + # Pase the raw data + while index < len(output) and 'ChannelId' in output[index]: + item = output[index][output[index].find('ChannelId'):].split(',') + # Channel ID + frame_results = {} + channel = item[0].split(':') + if channel[1] not in self.results[device][case_index]: + self.results[device][case_index][channel[1]] = frame_results + + # Frame ID + object_results = {} + frame = item[1].split(':') + if frame[1] not in self.results[device][case_index][channel[1]]: + self.results[device][case_index][channel[1]][frame[1]] = object_results + + # Object ID + label_prob_pos_results = [] + objid = item[2].split(':') + if objid[1] not in self.results[device][case_index][channel[1]][frame[1]]: + self.results[device][case_index][channel[1]][frame[1]][objid[1]] = label_prob_pos_results + self.results[device][case_index][channel[1]][frame[1]][objid[1]] = item[3:] + index += 1 + + index += 1 + + + def checker(self): + self.parser() + for device in self.results: + for case in self.results[device]: + print ("---* Device: {} - Case: {} *----\n".format(device, case)) + for channel in self.results[device][case]: + print ("Channel: {} - :{}".format(channel, self.results[device][case][channel])) + print ('---------------------------------------------------------') + + flag = False + if "CPU" in self.results and "AUTO:CPU" in self.results: + if self.results['CPU'] == self.results['AUTO:CPU']: + flag = True + else: + print ("CPU vs AUTO:CPU have inconsistent results") + + if "GPU" in self.results and "AUTO:GPU" in self.results: + if self.results['GPU'] == self.results['AUTO:GPU']: + flag = True + else: + print ("GPU vs AUTO:GPU have inconsistent results") + + return flag + + + +DEMOS = [ + CPPDemo(name='security_barrier_camera_demo') +] +def main(): + for demo in DEMOS: + print ("Checking {}...".format(demo.name)) + if demo.checker(): + print("Demo: {} passed.".format(demo.name)) + else: + print("Demo: {} failed.".format(demo.name)) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index f1da11f8bd1..d1cdb337b20 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -253,7 +253,12 @@ def main(): print('Testing {}...'.format(demo.subdirectory)) print() demo.set_precisions(args.precisions, model_info) - + filename = '/tmp/' + demo.subdirectory + os.makedirs(filename, exist_ok=True) + fo = open(filename + '/results.log', 'w+') + print("Save to {}".format(filename)) + content = '' + content += 'Testing {}...'.format(demo.subdirectory) + '\n' declared_model_names = set() for model_data in json.loads(subprocess.check_output( [sys.executable, '--', str(auto_tools_dir / 'info_dumper.py'), @@ -316,30 +321,34 @@ def option_to_args(key, value): print('Test case #{}/{}:'.format(test_case_index, device), ' '.join(shlex.quote(str(arg)) for arg in dev_arg + case_args)) print(flush=True) + content += "Device:{}\nCaseId:{}\n".format(device, test_case_index) + rawResults = '' + execution_time = -1 try: start_time = timeit.default_timer() output = subprocess.check_output(fixed_args + dev_arg + case_args, stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf-8', env=demo_environment) execution_time = timeit.default_timer() - start_time + for line in output.split('\n'): + if "DEBUG" in line: + rawResults += line + rawResults += '\n' demo.parse_output(output, device, test_case_index) except subprocess.CalledProcessError as e: print(e.output) print('Exit code:', e.returncode) num_failures += 1 execution_time = -1 + rawResults = {} + content += "Execution_time:{}\n".format(execution_time) + content += "{}\n".format(rawResults) + fo.write(content) + content = '' if args.report_file: collect_result(demo.subdirectory, device, case_model_names, execution_time, args.report_file) - if not demo.results_correctness_check(): - print ('Results Correctness checking failed for Demo: ', demo.subdirectory) - else: - print ('Results Correctness checking Passed') - - - - - print() + fo.close() print("Failures: {}".format(num_failures)) From 6fb10ef4eab1c30c1c847352432a0e7b5077fc65 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Tue, 22 Mar 2022 17:11:48 +0800 Subject: [PATCH 07/46] . exit when task list is empty and inputs source is image instead of video. 2 Update case parameters in order to capturing ROI data from this demo. 3 Enable parse function for AUTO device. Signed-off-by: Wang, Yang --- demos/common/cpp/utils/include/utils/threads_common.hpp | 3 ++- demos/common/cpp/utils/src/args_helper.cpp | 2 +- demos/security_barrier_camera_demo/cpp/main.cpp | 3 ++- demos/tests/cases.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/demos/common/cpp/utils/include/utils/threads_common.hpp b/demos/common/cpp/utils/include/utils/threads_common.hpp index f0e5cbf3dec..2b4e50e187e 100644 --- a/demos/common/cpp/utils/include/utils/threads_common.hpp +++ b/demos/common/cpp/utils/include/utils/threads_common.hpp @@ -100,7 +100,8 @@ class Worker { } } void stop() { - running = false; + if (tasks.empty()) + running = false; tasksCondVar.notify_all(); } void join() { diff --git a/demos/common/cpp/utils/src/args_helper.cpp b/demos/common/cpp/utils/src/args_helper.cpp index 8f4bc351d61..b55d1f44bf2 100644 --- a/demos/common/cpp/utils/src/args_helper.cpp +++ b/demos/common/cpp/utils/src/args_helper.cpp @@ -81,7 +81,7 @@ std::vector parseDevices(const std::string& device_string) { const std::string::size_type colon_position = device_string.find(":"); if (colon_position != std::string::npos) { std::string device_type = device_string.substr(0, colon_position); - if (device_type == "HETERO" || device_type == "MULTI") { + if (device_type == "HETERO" || device_type == "MULTI" || device_type == "AUTO") { std::string comma_separated_devices = device_string.substr(colon_position + 1); std::vector devices = split(comma_separated_devices, ','); for (auto& device : devices) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index a05025722ac..b8c2940faa5 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -292,6 +292,8 @@ ReborningVideoFrame::~ReborningVideoFrame() { context.videoFramesContext.lastFrameIdsMutexes[sourceID].lock(); const auto frameId = ++context.videoFramesContext.lastframeIds[sourceID]; context.videoFramesContext.lastFrameIdsMutexes[sourceID].unlock(); + if (!context.isVideo && frameId >= FLAGS_n_iqs) + return; std::shared_ptr reborn = std::make_shared(context, sourceID, frameId, frame); worker->push(std::make_shared(reborn)); } catch (const std::bad_weak_ptr&) {} @@ -635,7 +637,6 @@ bool Reader::isReady() { } void Reader::process() { - //slog::debug <<"Reader::process for channel-frameid [" << sharedVideoFrame->sourceID <<" - " << sharedVideoFrame->frameId << "]....." << slog::endl; unsigned sourceID = sharedVideoFrame->sourceID; sharedVideoFrame->timestamp = std::chrono::steady_clock::now(); Context& context = static_cast(sharedVideoFrame.get())->context; diff --git a/demos/tests/cases.py b/demos/tests/cases.py index 860e7dedebc..dd4b0ef3f6a 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -594,7 +594,7 @@ def single_option_cases(key, *args): TestCase(options={'-no_show': None, '-r' : None, **MONITORS, '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}), - TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0106')}), + TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0123')}), single_option_cases('-m_lpr', None, ModelArg('license-plate-recognition-barrier-0001'), From 7eca858cfbd98034d406fa067a6fc07db5e5bd0b Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 23 Mar 2022 10:05:29 +0800 Subject: [PATCH 08/46] Exit worker thread when the inferences of all frames have been completed. Signed-off-by: Wang, Yang --- demos/security_barrier_camera_demo/cpp/main.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index b8c2940faa5..98694009a08 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -292,6 +292,8 @@ ReborningVideoFrame::~ReborningVideoFrame() { context.videoFramesContext.lastFrameIdsMutexes[sourceID].lock(); const auto frameId = ++context.videoFramesContext.lastframeIds[sourceID]; context.videoFramesContext.lastFrameIdsMutexes[sourceID].unlock(); + + // Stop reborning when frame ID reached to input queue size if (!context.isVideo && frameId >= FLAGS_n_iqs) return; std::shared_ptr reborn = std::make_shared(context, sourceID, frameId, frame); @@ -335,7 +337,6 @@ bool Drawer::isReady() { void Drawer::process() { const int64_t frameId = sharedVideoFrame->frameId; - //slog::debug <<"Drawer::process for channel-frameid [" << sharedVideoFrame->sourceID <<" - " << sharedVideoFrame->frameId << "]....." << slog::endl; Context& context = static_cast(sharedVideoFrame.get())->context; std::map& gridMats = context.drawersContext.gridMats; context.drawersContext.drawerMutex.lock(); @@ -382,7 +383,9 @@ void Drawer::process() { } else { if (!context.isVideo) { try { - std::shared_ptr(context.drawersContext.drawersWorker)->stop(); + // Exit only when inferences on all of frames are finished. + if (context.frameCounter >= FLAGS_n_iqs * context.readersContext.inputChannels.size()) + std::shared_ptr(context.drawersContext.drawersWorker)->stop(); } catch (const std::bad_weak_ptr&) {} } @@ -486,7 +489,6 @@ bool DetectionsProcessor::isReady() { } void DetectionsProcessor::process() { - //slog::debug <<"DetectionProcessor::process for channel-frameid [" << sharedVideoFrame->sourceID <<" - " << sharedVideoFrame->frameId << "]....." << slog::endl; Context& context = static_cast(sharedVideoFrame.get())->context; if (!FLAGS_m_va.empty()) { auto vehicleRectsIt = vehicleRects.begin(); From a2f84217bc0c05a28cd8a9ff224237f8ab554dd8 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 23 Mar 2022 10:20:12 +0800 Subject: [PATCH 09/46] Update parameters of demo security_barrier_camera_demo so that just inputing single iamge for each channel. Signed-off-by: Wang, Yang --- demos/tests/cases.py | 2 +- demos/tests/correctness_checker.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/demos/tests/cases.py b/demos/tests/cases.py index dd4b0ef3f6a..656c74bb65e 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -591,7 +591,7 @@ def single_option_cases(key, *args): model_keys=['-m', '-m_lpr', '-m_va'], device_keys=['-d', '-d_lpr', '-d_va'], test_cases=combine_cases( - TestCase(options={'-no_show': None, '-r' : None, + TestCase(options={'-no_show': None, '-r' : None, '-n_iqs': '1', **MONITORS, '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}), TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0123')}), diff --git a/demos/tests/correctness_checker.py b/demos/tests/correctness_checker.py index 590b23397ea..b09ff76459c 100755 --- a/demos/tests/correctness_checker.py +++ b/demos/tests/correctness_checker.py @@ -94,12 +94,6 @@ def parser(self): def checker(self): self.parser() - for device in self.results: - for case in self.results[device]: - print ("---* Device: {} - Case: {} *----\n".format(device, case)) - for channel in self.results[device][case]: - print ("Channel: {} - :{}".format(channel, self.results[device][case][channel])) - print ('---------------------------------------------------------') flag = False if "CPU" in self.results and "AUTO:CPU" in self.results: @@ -114,6 +108,13 @@ def checker(self): else: print ("GPU vs AUTO:GPU have inconsistent results") + if not flag: + for device in self.results: + for case in self.results[device]: + print ("---* Device: {} - Case: {} *----\n".format(device, case)) + for channel in self.results[device][case]: + print ("Channel: {} - :{}".format(channel, self.results[device][case][channel])) + print ('---------------------------------------------------------') return flag From cc86f1df4df6487bb8a60307f61cf8cca0684bd6 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 23 Mar 2022 10:37:01 +0800 Subject: [PATCH 10/46] Add the comment of replacing model for the demo security_barrier_camera_demo. Signed-off-by: Wang, Yang --- demos/tests/cases.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/demos/tests/cases.py b/demos/tests/cases.py index c26a903a768..54f067027f1 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -591,9 +591,10 @@ def single_option_cases(key, *args): model_keys=['-m', '-m_lpr', '-m_va'], device_keys=['-d', '-d_lpr', '-d_va'], test_cases=combine_cases( - TestCase(options={'-no_show': None, '-r' : None, '-n_iqs': '1', + TestCase(options={'-no_show': None, '-r' : None, '-n_iqs': '2', **MONITORS, '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}), + # Change the model in order to obtain the ROI data TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0123')}), single_option_cases('-m_lpr', None, From 37f8371428c84a2ae8c74f1964077fa1c85e8fc1 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 23 Mar 2022 10:43:29 +0800 Subject: [PATCH 11/46] Input single image for demo security_barrier_camera_demo. Signed-off-by: Wang, Yang --- demos/tests/cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/tests/cases.py b/demos/tests/cases.py index 54f067027f1..c565cd206bc 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -591,7 +591,7 @@ def single_option_cases(key, *args): model_keys=['-m', '-m_lpr', '-m_va'], device_keys=['-d', '-d_lpr', '-d_va'], test_cases=combine_cases( - TestCase(options={'-no_show': None, '-r' : None, '-n_iqs': '2', + TestCase(options={'-no_show': None, '-r' : None, '-n_iqs': '1', **MONITORS, '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}), # Change the model in order to obtain the ROI data From 237dcf3c776cc1460351d42c076045dd275f2e98 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 23 Mar 2022 10:46:09 +0800 Subject: [PATCH 12/46] Update. Signed-off-by: Wang, Yang --- demos/security_barrier_camera_demo/cpp/main.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index 98694009a08..30899ddbf4d 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -398,7 +398,6 @@ void Drawer::process() { } void ResAggregator::process() { - //slog::debug <<"ResAggregator::process for channel-frameid [" << sharedVideoFrame->sourceID <<" - " << sharedVideoFrame->frameId << "]....." << slog::endl; Context& context = static_cast(sharedVideoFrame.get())->context; context.freeDetectionInfersCount += context.detectorsInfers.inferRequests.lockedSize(); context.frameCounter++; From d0e831b5e62b3da2f6f0358a274fc3ae292dfe81 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 23 Mar 2022 10:52:45 +0800 Subject: [PATCH 13/46] Update. Signed-off-by: Wang, Yang --- demos/security_barrier_camera_demo/cpp/main.cpp | 17 ----------------- .../cpp/net_wrappers.hpp | 13 ------------- 2 files changed, 30 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index 30899ddbf4d..4f0f3ae021a 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -592,21 +592,6 @@ void InferTask::process() { std::reference_wrapper inferRequest = detectorsInfers.inferRequests.container.back(); detectorsInfers.inferRequests.container.pop_back(); detectorsInfers.inferRequests.mutex.unlock(); - - //for (int i = 0; i < sharedVideoFrame->frame.rows; i++) { - // if(i == sharedVideoFrame->frame.rows - 1) - // slog::debug <<"InferTask::process for channel-frameid-imgsize [" << sharedVideoFrame->sourceID <<" - " << sharedVideoFrame->frameId << "-" << sharedVideoFrame->frame.total() << "]....." << slog::endl; - // for (int j = 0; j < sharedVideoFrame->frame.cols; j++) { - // auto bgr = sharedVideoFrame->frame.at(i,j); - // int a = bgr[0]; - // int b = bgr[1]; - // int c = bgr[2]; - // if(i == sharedVideoFrame->frame.rows - 1) - // std::cout << a <<" " << b << " " << c << " "; - // } - // if(i == sharedVideoFrame->frame.rows - 1) - // std::cout << std::endl; - //} context.inferTasksContext.detector.setImage(inferRequest, sharedVideoFrame->frame); inferRequest.get().set_callback( @@ -828,12 +813,10 @@ int main(int argc, char* argv[]) { context.drawersContext.drawersWorker = worker; context.resAggregatorsWorker = worker; - int readerCount = 0; for (uint64_t i = 0; i < FLAGS_n_iqs; i++) { for (unsigned sourceID = 0; sourceID < inputChannels.size(); sourceID++) { VideoFrame::Ptr sharedVideoFrame = std::make_shared(context, sourceID, i); worker->push(std::make_shared(sharedVideoFrame)); - readerCount++; } } diff --git a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp index a52370080db..f8810491fbc 100644 --- a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp +++ b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp @@ -119,19 +119,6 @@ class Detector { // resize and copy data from image to tensor using OpenCV resize2tensor(img, inputTensor); } - /* - InferenceEngine::SizeVector blobSize = input->getTensorDesc().getDims(); - const size_t width = blobSize[3]; - const size_t height = blobSize[2]; - const size_t channels = blobSize[1]; - InferenceEngine::LockedMemory blobMapped = InferenceEngine::as(input)->wmap(); - uint8_t* blobData = blobMapped.as(); - for (size_t c = 0; c < channels; c++) - for (size_t h = 0; h < height; h++) - for (size_t w = 0; w < width; w++) - std::cout << blobData[ c * width * height + h * width + w]; - std::cout << std::endl; - */ } std::list getResults(ov::InferRequest& inferRequest, const int64_t channelID, const int64_t frameID, cv::Size upscale, std::vector& rawResults) { From 6b9ce5a2423e2b8dfe45df0a843751edda3e011f Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Fri, 25 Mar 2022 13:05:06 +0800 Subject: [PATCH 14/46] Update. Signed-off-by: Wang, Yang --- demos/tests/correctness_checker.py | 70 ++++++++++++++---------------- demos/tests/run_tests.py | 10 ++--- 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/demos/tests/correctness_checker.py b/demos/tests/correctness_checker.py index b09ff76459c..e3186a531b2 100755 --- a/demos/tests/correctness_checker.py +++ b/demos/tests/correctness_checker.py @@ -15,7 +15,6 @@ # limitations under the License. from abc import ABC, abstractmethod -from unittest import case class Demo(ABC): def __init__(self, name, implementation): @@ -23,10 +22,10 @@ def __init__(self, name, implementation): self.implementation = implementation self.subdirectory = name + '/' + implementation self.results = {} - + @abstractmethod def parser(self): - pass + pass @abstractmethod def checker(self): @@ -58,14 +57,14 @@ def parser(self): case_index = output[index][output[index].find(":") + 1:] if case_index not in self.results[device]: self.results[device][case_index] = {} - + if "Execution_time" in output[index]: execution_time = output[index].split(':')[1] if execution_time == '-1': - while index < len(output) and 'Device' not in output[index]: - index += 1 - continue - + while index < len(output) and 'Device' not in output[index]: + index += 1 + continue + # Pase the raw data while index < len(output) and 'ChannelId' in output[index]: item = output[index][output[index].find('ChannelId'):].split(',') @@ -91,45 +90,40 @@ def parser(self): index += 1 - def checker(self): self.parser() - flag = False - if "CPU" in self.results and "AUTO:CPU" in self.results: - if self.results['CPU'] == self.results['AUTO:CPU']: - flag = True - else: - print ("CPU vs AUTO:CPU have inconsistent results") - - if "GPU" in self.results and "AUTO:GPU" in self.results: - if self.results['GPU'] == self.results['AUTO:GPU']: - flag = True - else: - print ("GPU vs AUTO:GPU have inconsistent results") - - if not flag: - for device in self.results: - for case in self.results[device]: - print ("---* Device: {} - Case: {} *----\n".format(device, case)) - for channel in self.results[device][case]: - print ("Channel: {} - :{}".format(channel, self.results[device][case][channel])) - print ('---------------------------------------------------------') + flag = True + devices_list = {"CPU" : ["AUTO:CPU", "MULTI:CPU"], "GPU" : ["AUTO:CPU", "MULTI:CPU"]} + for device in devices_list: + for target in devices_list[device]: + if device in self.results and target in self.results: + if self.results[device] != self.results[target]: + flag = False + print("Failed: {}-{} have inconsistent results".format(device, target)) + # Show the detailed inconsistent results + for case in self.results[target]: + if self.results[device][case] != self.results[target][case]: + print("---* Device: {} - Case: {} *----\n".format(device, case)) + for channel in self.results[device][case]: + print("Channel: {} - :{}".format(channel, self.results[device][case][channel])) + print('---------------------------------------------------------') + print("---* Device: {} - Case: {} *----\n".format(target, case)) + for channel in self.results[target][case]: + print("Channel: {} - :{}".format(channel, self.results[target][case][channel])) + print('---------------------------------------------------------') return flag - - DEMOS = [ CPPDemo(name='security_barrier_camera_demo') ] def main(): - for demo in DEMOS: - print ("Checking {}...".format(demo.name)) - if demo.checker(): - print("Demo: {} passed.".format(demo.name)) - else: - print("Demo: {} failed.".format(demo.name)) - + for demo in DEMOS: + print("Checking {}...".format(demo.name)) + if demo.checker(): + print("Demo {} correctness checking: Passed.".format(demo.name)) + else: + print("Demo {} correctness checking: Failed.".format(demo.name)) if __name__ == '__main__': main() \ No newline at end of file diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index 4535bb14eb7..cc6a8c2f605 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -266,7 +266,7 @@ def main(): os.makedirs(filename, exist_ok=True) fo = open(filename + '/results.log', 'w+') print("Save to {}".format(filename)) - content = '' + content = '' content += 'Testing {}...'.format(demo.subdirectory) + '\n' declared_model_names = set() for model_data in json.loads(subprocess.check_output( @@ -332,8 +332,8 @@ def option_to_args(key, value): print(test_descr) print(flush=True) content += "Device:{}\nCaseId:{}\n".format(device, test_case_index) - rawResults = '' - execution_time = -1 + rawResults = '' + execution_time = -1 try: start_time = timeit.default_timer() output = subprocess.check_output(fixed_args + dev_arg + case_args, @@ -344,7 +344,7 @@ def option_to_args(key, value): if "DEBUG" in line: rawResults += line rawResults += '\n' - demo.parse_output(output, device, test_case_index) + demo.parse_output(output, test_case, device) except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: output = e.output if isinstance(e, subprocess.CalledProcessError): @@ -357,7 +357,7 @@ def option_to_args(key, value): num_failures += 1 execution_time = -1 rawResults = {} - + content += "Execution_time:{}\n".format(execution_time) content += "{}\n".format(rawResults) fo.write(content) From 85327efc2da6a991f50ff4ac550aefa499827535 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Fri, 25 Mar 2022 16:14:20 +0800 Subject: [PATCH 15/46] Decouple of the raw data saving from the run_tests.py. Signed-off-by: Wang, Yang --- demos/tests/correctness_checker.py | 64 ++++++++++++++++-------------- demos/tests/run_tests.py | 12 +++--- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/demos/tests/correctness_checker.py b/demos/tests/correctness_checker.py index e3186a531b2..ecedc916752 100755 --- a/demos/tests/correctness_checker.py +++ b/demos/tests/correctness_checker.py @@ -43,7 +43,7 @@ def parser(self): fo = open('/tmp/' + self.subdirectory + '/results.log', 'r') except IOError as err: print("File error: " + str(err)) - output = [i.rstrip() for i in fo.readlines()] + output = [i.rstrip() for i in fo.readlines() if "DEBUG" in i] device = '' case_index = '' index = 1 @@ -52,49 +52,53 @@ def parser(self): device = output[index][output[index].find(":") + 1:] if device not in self.results: self.results[device] = {} - - if "CaseId" in output[index]: + index += 1 + continue + elif "CaseId" in output[index]: case_index = output[index][output[index].find(":") + 1:] if case_index not in self.results[device]: self.results[device][case_index] = {} + index += 1 + continue - if "Execution_time" in output[index]: + elif "Execution_time" in output[index]: execution_time = output[index].split(':')[1] if execution_time == '-1': while index < len(output) and 'Device' not in output[index]: index += 1 continue - - # Pase the raw data - while index < len(output) and 'ChannelId' in output[index]: - item = output[index][output[index].find('ChannelId'):].split(',') - # Channel ID - frame_results = {} - channel = item[0].split(':') - if channel[1] not in self.results[device][case_index]: - self.results[device][case_index][channel[1]] = frame_results - - # Frame ID - object_results = {} - frame = item[1].split(':') - if frame[1] not in self.results[device][case_index][channel[1]]: - self.results[device][case_index][channel[1]][frame[1]] = object_results - - # Object ID - label_prob_pos_results = [] - objid = item[2].split(':') - if objid[1] not in self.results[device][case_index][channel[1]][frame[1]]: - self.results[device][case_index][channel[1]][frame[1]][objid[1]] = label_prob_pos_results - self.results[device][case_index][channel[1]][frame[1]][objid[1]] = item[3:] index += 1 - - index += 1 + continue + elif 'ChannelId' in output[index]: + # Pase the raw data + while index < len(output) and 'ChannelId' in output[index]: + item = output[index][output[index].find('ChannelId'):].split(',') + # Channel ID + frame_results = {} + channel = item[0].split(':') + if channel[1] not in self.results[device][case_index]: + self.results[device][case_index][channel[1]] = frame_results + + # Frame ID + object_results = {} + frame = item[1].split(':') + if frame[1] not in self.results[device][case_index][channel[1]]: + self.results[device][case_index][channel[1]][frame[1]] = object_results + + # Object ID + label_prob_pos_results = [] + objid = item[2].split(':') + if objid[1] not in self.results[device][case_index][channel[1]][frame[1]]: + self.results[device][case_index][channel[1]][frame[1]][objid[1]] = label_prob_pos_results + self.results[device][case_index][channel[1]][frame[1]][objid[1]] = item[3:] + index += 1 + else: + index += 1 def checker(self): self.parser() - flag = True - devices_list = {"CPU" : ["AUTO:CPU", "MULTI:CPU"], "GPU" : ["AUTO:CPU", "MULTI:CPU"]} + devices_list = {"CPU" : ["AUTO:CPU", "MULTI:CPU", "AUTO:CPU,GPU"], "GPU" : ["AUTO:CPU", "MULTI:CPU", "AUTO:CPU,GPU"]} for device in devices_list: for target in devices_list[device]: if device in self.results and target in self.results: diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index cc6a8c2f605..5c3cdde5c9b 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -267,7 +267,7 @@ def main(): fo = open(filename + '/results.log', 'w+') print("Save to {}".format(filename)) content = '' - content += 'Testing {}...'.format(demo.subdirectory) + '\n' + content += '[ DEBUG ] Testing {}...'.format(demo.subdirectory) + '\n' declared_model_names = set() for model_data in json.loads(subprocess.check_output( [sys.executable, '--', str(auto_tools_dir / 'info_dumper.py'), @@ -331,7 +331,7 @@ def option_to_args(key, value): for arg in fixed_args + dev_arg + case_args)) print(test_descr) print(flush=True) - content += "Device:{}\nCaseId:{}\n".format(device, test_case_index) + content += "[ DEBUG ] Device:{}\n[ DEBUG ] CaseId:{}\n".format(device, test_case_index) rawResults = '' execution_time = -1 try: @@ -340,10 +340,8 @@ def option_to_args(key, value): stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf-8', env=demo_environment, timeout=600) execution_time = timeit.default_timer() - start_time - for line in output.split('\n'): - if "DEBUG" in line: - rawResults += line - rawResults += '\n' + for line in output: + rawResults += line demo.parse_output(output, test_case, device) except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: output = e.output @@ -358,7 +356,7 @@ def option_to_args(key, value): execution_time = -1 rawResults = {} - content += "Execution_time:{}\n".format(execution_time) + content += "[ DEBUG ] Execution_time:{}\n".format(execution_time) content += "{}\n".format(rawResults) fo.write(content) content = '' From ab9d19cae56b421f24cf27c2076e86c50daaa400 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Mon, 28 Mar 2022 13:42:54 +0800 Subject: [PATCH 16/46] Update. Signed-off-by: Wang, Yang --- demos/face_recognition_demo/python/face_recognition_demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/face_recognition_demo/python/face_recognition_demo.py b/demos/face_recognition_demo/python/face_recognition_demo.py index 1b48ed2687d..be777c7edbb 100755 --- a/demos/face_recognition_demo/python/face_recognition_demo.py +++ b/demos/face_recognition_demo/python/face_recognition_demo.py @@ -42,7 +42,7 @@ log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.DEBUG, stream=sys.stdout) -DEVICE_KINDS = ['CPU', 'GPU', 'AUTO', 'AUTO:CPU,GPU', 'MULTI:CPU,GPU', 'MYRIAD', 'HETERO', 'HDDL'] +DEVICE_KINDS = ['CPU', 'GPU', 'MYRIAD', 'HETERO', 'HDDL'] def build_argparser(): From f1c5c8730ce6146620f8ec23ccf8569a0664e5ee Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Thu, 31 Mar 2022 16:02:52 +0800 Subject: [PATCH 17/46] Add scope 'correctness' to enable correctness checking. Signed-off-by: Wang, Yang --- demos/tests/cases.py | 6 +- demos/tests/correctness_checker_cases.py | 155 +++++++++++++++++++++++ demos/tests/run_tests.py | 6 +- 3 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 demos/tests/correctness_checker_cases.py diff --git a/demos/tests/cases.py b/demos/tests/cases.py index c565cd206bc..3909a88d0f2 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -73,6 +73,10 @@ def parse_output(self, output, test_case, device): if self.parser: self.parser(output, test_case, device) + def check_difference(self): + if self.parser: + return self.parser.check_difference() + def update_option(self, updated_options): for case in self.test_cases[:]: self.update_case(case, updated_options, with_replacement=True) @@ -591,7 +595,7 @@ def single_option_cases(key, *args): model_keys=['-m', '-m_lpr', '-m_va'], device_keys=['-d', '-d_lpr', '-d_va'], test_cases=combine_cases( - TestCase(options={'-no_show': None, '-r' : None, '-n_iqs': '1', + TestCase(options={'-no_show': None, **MONITORS, '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}), # Change the model in order to obtain the ROI data diff --git a/demos/tests/correctness_checker_cases.py b/demos/tests/correctness_checker_cases.py new file mode 100644 index 00000000000..e8326b08ea7 --- /dev/null +++ b/demos/tests/correctness_checker_cases.py @@ -0,0 +1,155 @@ +# Copyright (c) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +from args import DataPatternArg +from copy import deepcopy +from abc import ABC, abstractmethod + +from cases import BASE, single_option_cases + +THREADS_NUM = os.cpu_count() + + +class CorrectnessCheckerBase(ABC): + def __init__(self, demo): + self.filename = demo.subdirectory.replace('/', '_') + '.log' + self.demo_name = demo.subdirectory.split('/')[0] + self.results = {} + self.case_index = {} + + @abstractmethod + def __call__(self, output, test_case, device, execution_time=-1): + print("========\nOutput: {}\n======\ntest_case: {}\n========\ndevice: {} \n==========".format(output, test_case, device)) + + @abstractmethod + def check_difference(self): + pass + + def write_to_log(self, result, test_case, device): + with open(self.filename, 'w') as f: + print(self.results, file=f) + + +class DemoSecurityBarrierCamera(CorrectnessCheckerBase): + def __init__(self, demo): + super().__init__(demo) + + def __call__(self, output, test_case, device, execution_time=-1): + # Results format + # {"device name": + # {"case index 0": + # {"channel id 0": + # {"frame id 0": + # {"object id 0":{"label:xx,prob:xx,x,y,width,hight"}, + # {"object id 1":{"label:xx,prob:xx,x,y,width,hight"}, + # ..................... + # {"object id n":{"label:xx,prob:xx,x,y,width,hight"} + # }, + # ..... + # {"frame id n": + # ..... + # } + # }, + # ..... + # {"channel id n": + # ..... + # } + # {"case index n": + # ..... + # } + # } + + + # Generate case id for each device + if device not in self.case_index: + self.case_index[device] = 0 + + if device not in self.results: + self.results[device] = {} + + case_index = self.case_index[device] + if case_index not in self.results[device]: + self.results[device][case_index] = {} + + if execution_time < 0: + self.case_index[device] += 1 + return + + # Parsing the raw data + print("Demo {} results parsing....".format(self.demo_name)) + output = [i.rstrip() for i in output.split('\n') if "DEBUG" in i and "ChannelId" in i] + #print("========\nCase id: {}\nOutput: {}\n========".format(self.case_index[device], output)) + for item in output: + item = item[item.find('ChannelId'):].split(',') + # Channel ID + frame_results = {} + channel = item[0].split(':')[1] + if channel not in self.results[device][case_index]: + self.results[device][case_index][channel] = frame_results + + # Frame ID + object_results = {} + frame = item[1].split(':')[1] + if frame not in self.results[device][case_index][channel]: + self.results[device][case_index][channel][frame] = object_results + + # Object ID + label_prob_pos_results = [] + objid = item[2].split(':')[1] + if objid not in self.results[device][case_index][channel][frame]: + self.results[device][case_index][channel][frame][objid] = label_prob_pos_results + self.results[device][case_index][channel][frame][objid] = item[3:] + + self.case_index[device] += 1 + + def check_difference(self): + flag = True + devices_list = {"AUTO:GPU,CPU" : ["CPU", "GPU"], + "AUTO:CPU" : ["CPU"], + "AUTO:GPU" : ["GPU"], + "MULTI:GPU,CPU" : ["CPU", "GPU"]} + err_msg = '' + for device in devices_list: + for target in devices_list[device]: + if device not in self.results or target not in self.results: + flag = False + err_msg += "\tMiss the results of device {} or device {}.\n".format(device,target) + if device in self.results and target in self.results: + if self.results[device] != self.results[target]: + flag = False + err_msg += "\tInconsistent results between device {} and {} \n".format(device, target) + # Show the detailed inconsistent results + for case in self.results[target]: + if self.results[device][case] != self.results[target][case]: + err_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(device, case)) + for channel in self.results[device][case]: + for frame in self.results[device][case][channel]: + err_msg += ("\t\t\tChannel {} - Frame {} : {}\n".format(channel, frame, self.results[device][case][channel][frame])) + err_msg += ('\t\t---------------------------------------------------------\n') + err_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(target, case)) + for channel in self.results[target][case]: + for frame in self.results[target][case][channel]: + err_msg += ("\t\t\tChannel {} - Frame {} : {}\n".format(channel, frame, self.results[target][case][channel][frame])) + err_msg += ('\t\t---------------------------------------------------------\n') + if not flag: + print("Correctness checking: Failure\n{}".format(err_msg)) + return flag + +DEMOS = [ + deepcopy(BASE['security_barrier_camera_demo/cpp']) + .update_option({'-r': None, '-n_iqs': '1'}) + .add_parser(DemoSecurityBarrierCamera) +] \ No newline at end of file diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index 5c3cdde5c9b..f51b67f7762 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -51,6 +51,7 @@ scopes = { 'base': importlib.import_module('cases').DEMOS, 'performance': importlib.import_module('performance_cases').DEMOS, + 'correctness': importlib.import_module('correctness_checker_cases').DEMOS, } @@ -72,7 +73,7 @@ def parse_args(): help='list of demos to run tests for (by default, every demo is tested). ' 'For testing demos of specific implementation pass one (or more) of the next values: cpp, cpp_gapi, python.') parser.add_argument('--scope', default='base', - help='The scenario for testing demos.', choices=('base', 'performance')) + help='The scenario for testing demos.', choices=('base', 'performance', 'correctness')) parser.add_argument('--mo', type=Path, metavar='MO.PY', help='Model Optimizer entry point script') parser.add_argument('--devices', default="CPU GPU", @@ -368,6 +369,9 @@ def option_to_args(key, value): write_log(test_descr, args.log_file) write_log(output, args.log_file) fo.close() + if args.scope == "correctness": + print("Demo {} correctness checking....".format(demo.subdirectory)) + demo.check_difference() print() print("{} failures:".format(num_failures)) From b4d110b799fc1b84592ae7b47d8fe54b24fab603 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Fri, 1 Apr 2022 14:54:03 +0800 Subject: [PATCH 18/46] Remove the log save for each demo and update the correctness checker. Signed-off-by: Wang, Yang --- ..._checker_cases.py => correctness_cases.py} | 3 +- demos/tests/correctness_checker.py | 133 ------------------ demos/tests/run_tests.py | 27 +--- 3 files changed, 8 insertions(+), 155 deletions(-) rename demos/tests/{correctness_checker_cases.py => correctness_cases.py} (98%) delete mode 100755 demos/tests/correctness_checker.py diff --git a/demos/tests/correctness_checker_cases.py b/demos/tests/correctness_cases.py similarity index 98% rename from demos/tests/correctness_checker_cases.py rename to demos/tests/correctness_cases.py index e8326b08ea7..71881ea9030 100644 --- a/demos/tests/correctness_checker_cases.py +++ b/demos/tests/correctness_cases.py @@ -47,7 +47,7 @@ class DemoSecurityBarrierCamera(CorrectnessCheckerBase): def __init__(self, demo): super().__init__(demo) - def __call__(self, output, test_case, device, execution_time=-1): + def __call__(self, output, test_case, device, execution_time=0): # Results format # {"device name": # {"case index 0": @@ -91,7 +91,6 @@ def __call__(self, output, test_case, device, execution_time=-1): # Parsing the raw data print("Demo {} results parsing....".format(self.demo_name)) output = [i.rstrip() for i in output.split('\n') if "DEBUG" in i and "ChannelId" in i] - #print("========\nCase id: {}\nOutput: {}\n========".format(self.case_index[device], output)) for item in output: item = item[item.find('ChannelId'):].split(',') # Channel ID diff --git a/demos/tests/correctness_checker.py b/demos/tests/correctness_checker.py deleted file mode 100755 index ecedc916752..00000000000 --- a/demos/tests/correctness_checker.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import ABC, abstractmethod - -class Demo(ABC): - def __init__(self, name, implementation): - self.name = name - self.implementation = implementation - self.subdirectory = name + '/' + implementation - self.results = {} - - @abstractmethod - def parser(self): - pass - - @abstractmethod - def checker(self): - pass - -class CPPDemo(Demo): - def __init__(self, name, implementation='cpp'): - super().__init__(name, implementation) - self.results = {} - pass - - def parser(self): - fo = None - try: - fo = open('/tmp/' + self.subdirectory + '/results.log', 'r') - except IOError as err: - print("File error: " + str(err)) - output = [i.rstrip() for i in fo.readlines() if "DEBUG" in i] - device = '' - case_index = '' - index = 1 - while index < len(output): - if "Device" in output[index]: - device = output[index][output[index].find(":") + 1:] - if device not in self.results: - self.results[device] = {} - index += 1 - continue - elif "CaseId" in output[index]: - case_index = output[index][output[index].find(":") + 1:] - if case_index not in self.results[device]: - self.results[device][case_index] = {} - index += 1 - continue - - elif "Execution_time" in output[index]: - execution_time = output[index].split(':')[1] - if execution_time == '-1': - while index < len(output) and 'Device' not in output[index]: - index += 1 - continue - index += 1 - continue - elif 'ChannelId' in output[index]: - # Pase the raw data - while index < len(output) and 'ChannelId' in output[index]: - item = output[index][output[index].find('ChannelId'):].split(',') - # Channel ID - frame_results = {} - channel = item[0].split(':') - if channel[1] not in self.results[device][case_index]: - self.results[device][case_index][channel[1]] = frame_results - - # Frame ID - object_results = {} - frame = item[1].split(':') - if frame[1] not in self.results[device][case_index][channel[1]]: - self.results[device][case_index][channel[1]][frame[1]] = object_results - - # Object ID - label_prob_pos_results = [] - objid = item[2].split(':') - if objid[1] not in self.results[device][case_index][channel[1]][frame[1]]: - self.results[device][case_index][channel[1]][frame[1]][objid[1]] = label_prob_pos_results - self.results[device][case_index][channel[1]][frame[1]][objid[1]] = item[3:] - index += 1 - else: - index += 1 - - def checker(self): - self.parser() - flag = True - devices_list = {"CPU" : ["AUTO:CPU", "MULTI:CPU", "AUTO:CPU,GPU"], "GPU" : ["AUTO:CPU", "MULTI:CPU", "AUTO:CPU,GPU"]} - for device in devices_list: - for target in devices_list[device]: - if device in self.results and target in self.results: - if self.results[device] != self.results[target]: - flag = False - print("Failed: {}-{} have inconsistent results".format(device, target)) - # Show the detailed inconsistent results - for case in self.results[target]: - if self.results[device][case] != self.results[target][case]: - print("---* Device: {} - Case: {} *----\n".format(device, case)) - for channel in self.results[device][case]: - print("Channel: {} - :{}".format(channel, self.results[device][case][channel])) - print('---------------------------------------------------------') - print("---* Device: {} - Case: {} *----\n".format(target, case)) - for channel in self.results[target][case]: - print("Channel: {} - :{}".format(channel, self.results[target][case][channel])) - print('---------------------------------------------------------') - return flag - -DEMOS = [ - CPPDemo(name='security_barrier_camera_demo') -] -def main(): - for demo in DEMOS: - print("Checking {}...".format(demo.name)) - if demo.checker(): - print("Demo {} correctness checking: Passed.".format(demo.name)) - else: - print("Demo {} correctness checking: Failed.".format(demo.name)) - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index f51b67f7762..fec2e29e7f6 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -51,7 +51,7 @@ scopes = { 'base': importlib.import_module('cases').DEMOS, 'performance': importlib.import_module('performance_cases').DEMOS, - 'correctness': importlib.import_module('correctness_checker_cases').DEMOS, + 'correctness': importlib.import_module('correctness_cases').DEMOS, } @@ -245,6 +245,7 @@ def main(): dl_dir = prepare_models(auto_tools_dir, args.downloader_cache_dir, args.mo, global_temp_dir, demos_to_test, args.precisions) num_failures = 0 + correctness_failures = 0 try: pythonpath = f"{os.environ['PYTHONPATH']}{os.pathsep}" @@ -263,12 +264,6 @@ def main(): print(header) print() demo.set_precisions(args.precisions, model_info) - filename = '/tmp/' + demo.subdirectory - os.makedirs(filename, exist_ok=True) - fo = open(filename + '/results.log', 'w+') - print("Save to {}".format(filename)) - content = '' - content += '[ DEBUG ] Testing {}...'.format(demo.subdirectory) + '\n' declared_model_names = set() for model_data in json.loads(subprocess.check_output( [sys.executable, '--', str(auto_tools_dir / 'info_dumper.py'), @@ -332,17 +327,12 @@ def option_to_args(key, value): for arg in fixed_args + dev_arg + case_args)) print(test_descr) print(flush=True) - content += "[ DEBUG ] Device:{}\n[ DEBUG ] CaseId:{}\n".format(device, test_case_index) - rawResults = '' - execution_time = -1 try: start_time = timeit.default_timer() output = subprocess.check_output(fixed_args + dev_arg + case_args, stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf-8', env=demo_environment, timeout=600) execution_time = timeit.default_timer() - start_time - for line in output: - rawResults += line demo.parse_output(output, test_case, device) except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: output = e.output @@ -354,13 +344,7 @@ def option_to_args(key, value): print(output) failed_tests.append(test_descr + '\n' + exit_msg) num_failures += 1 - execution_time = -1 - rawResults = {} - content += "[ DEBUG ] Execution_time:{}\n".format(execution_time) - content += "{}\n".format(rawResults) - fo.write(content) - content = '' if args.report_file: collect_result(demo.subdirectory, device, case_model_names, execution_time, args.report_file) if args.log_file: @@ -368,13 +352,16 @@ def option_to_args(key, value): write_log(header, args.log_file) write_log(test_descr, args.log_file) write_log(output, args.log_file) - fo.close() if args.scope == "correctness": print("Demo {} correctness checking....".format(demo.subdirectory)) - demo.check_difference() + if not demo.check_difference(): + correctness_failures += 1 print() print("{} failures:".format(num_failures)) + if args.scope == "correctness": + print("Correctness Failures: {}".format(correctness_failures)) + for test in failed_tests: print(test) From dbe9f13c3a0241c42725201ed6a5a4afa75ca55a Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Fri, 1 Apr 2022 15:00:17 +0800 Subject: [PATCH 19/46] Update. Signed-off-by: Wang, Yang --- demos/security_barrier_camera_demo/cpp/main.cpp | 2 +- demos/tests/cases.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index 4f0f3ae021a..d082a41f0d8 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -592,6 +592,7 @@ void InferTask::process() { std::reference_wrapper inferRequest = detectorsInfers.inferRequests.container.back(); detectorsInfers.inferRequests.container.pop_back(); detectorsInfers.inferRequests.mutex.unlock(); + context.inferTasksContext.detector.setImage(inferRequest, sharedVideoFrame->frame); inferRequest.get().set_callback( @@ -613,7 +614,6 @@ void InferTask::process() { bool Reader::isReady() { Context& context = static_cast(sharedVideoFrame.get())->context; context.readersContext.lastCapturedFrameIdsMutexes[sharedVideoFrame->sourceID].lock(); - // Look for the next frame if (context.readersContext.lastCapturedFrameIds[sharedVideoFrame->sourceID] + 1 == sharedVideoFrame->frameId) { return true; } else { diff --git a/demos/tests/cases.py b/demos/tests/cases.py index 3909a88d0f2..81bc4a7c370 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -598,7 +598,7 @@ def single_option_cases(key, *args): TestCase(options={'-no_show': None, **MONITORS, '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}), - # Change the model in order to obtain the ROI data + # Update to this model in order to obtain the ROI data TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0123')}), single_option_cases('-m_lpr', None, From 065ca0778172d71d702096baa2095ec0eda55e4a Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Fri, 1 Apr 2022 15:19:15 +0800 Subject: [PATCH 20/46] Update format and remove some redundant code. Signed-off-by: Wang, Yang --- demos/tests/correctness_cases.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 71881ea9030..8db5292f702 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -13,12 +13,10 @@ # limitations under the License. import os -import re -from args import DataPatternArg from copy import deepcopy from abc import ABC, abstractmethod -from cases import BASE, single_option_cases +from cases import BASE THREADS_NUM = os.cpu_count() @@ -32,7 +30,7 @@ def __init__(self, demo): @abstractmethod def __call__(self, output, test_case, device, execution_time=-1): - print("========\nOutput: {}\n======\ntest_case: {}\n========\ndevice: {} \n==========".format(output, test_case, device)) + pass @abstractmethod def check_difference(self): @@ -51,7 +49,7 @@ def __call__(self, output, test_case, device, execution_time=0): # Results format # {"device name": # {"case index 0": - # {"channel id 0": + # {"channel id 0": # {"frame id 0": # {"object id 0":{"label:xx,prob:xx,x,y,width,hight"}, # {"object id 1":{"label:xx,prob:xx,x,y,width,hight"}, @@ -64,7 +62,7 @@ def __call__(self, output, test_case, device, execution_time=0): # } # }, # ..... - # {"channel id n": + # {"channel id n": # ..... # } # {"case index n": @@ -79,17 +77,16 @@ def __call__(self, output, test_case, device, execution_time=0): if device not in self.results: self.results[device] = {} - - case_index = self.case_index[device] + + case_index = self.case_index[device] if case_index not in self.results[device]: self.results[device][case_index] = {} - + if execution_time < 0: self.case_index[device] += 1 return # Parsing the raw data - print("Demo {} results parsing....".format(self.demo_name)) output = [i.rstrip() for i in output.split('\n') if "DEBUG" in i and "ChannelId" in i] for item in output: item = item[item.find('ChannelId'):].split(',') @@ -98,13 +95,13 @@ def __call__(self, output, test_case, device, execution_time=0): channel = item[0].split(':')[1] if channel not in self.results[device][case_index]: self.results[device][case_index][channel] = frame_results - + # Frame ID object_results = {} frame = item[1].split(':')[1] if frame not in self.results[device][case_index][channel]: self.results[device][case_index][channel][frame] = object_results - + # Object ID label_prob_pos_results = [] objid = item[2].split(':')[1] @@ -113,7 +110,7 @@ def __call__(self, output, test_case, device, execution_time=0): self.results[device][case_index][channel][frame][objid] = item[3:] self.case_index[device] += 1 - + def check_difference(self): flag = True devices_list = {"AUTO:GPU,CPU" : ["CPU", "GPU"], @@ -125,7 +122,7 @@ def check_difference(self): for target in devices_list[device]: if device not in self.results or target not in self.results: flag = False - err_msg += "\tMiss the results of device {} or device {}.\n".format(device,target) + err_msg += "\tMiss the results of device {} or device {}.\n".format(device, target) if device in self.results and target in self.results: if self.results[device] != self.results[target]: flag = False @@ -151,4 +148,4 @@ def check_difference(self): deepcopy(BASE['security_barrier_camera_demo/cpp']) .update_option({'-r': None, '-n_iqs': '1'}) .add_parser(DemoSecurityBarrierCamera) -] \ No newline at end of file +] From 3755135785f923569e570d72015018c20d978a4b Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Fri, 1 Apr 2022 15:22:20 +0800 Subject: [PATCH 21/46] Update. Signed-off-by: Wang, Yang --- demos/tests/correctness_cases.py | 1 - demos/tests/run_tests.py | 1 - 2 files changed, 2 deletions(-) diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 8db5292f702..cd54a602126 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -70,7 +70,6 @@ def __call__(self, output, test_case, device, execution_time=0): # } # } - # Generate case id for each device if device not in self.case_index: self.case_index[device] = 0 diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index fec2e29e7f6..f3394c3aef4 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -33,7 +33,6 @@ import contextlib import csv import json -from operator import contains import os import shlex import subprocess # nosec - disable B404:import-subprocess check From 603c120a657154ef9259e613c3849b478579392c Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Sat, 2 Apr 2022 09:57:25 +0800 Subject: [PATCH 22/46] Revert the common thread. Signed-off-by: Wang, Yang --- demos/common/cpp/utils/include/utils/threads_common.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/demos/common/cpp/utils/include/utils/threads_common.hpp b/demos/common/cpp/utils/include/utils/threads_common.hpp index 2b4e50e187e..f0e5cbf3dec 100644 --- a/demos/common/cpp/utils/include/utils/threads_common.hpp +++ b/demos/common/cpp/utils/include/utils/threads_common.hpp @@ -100,8 +100,7 @@ class Worker { } } void stop() { - if (tasks.empty()) - running = false; + running = false; tasksCondVar.notify_all(); } void join() { From 2e2ade492f1f7cb21ad2e2d9df500a3c4cb3ee1f Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 6 Apr 2022 10:24:33 +0800 Subject: [PATCH 23/46] Update. Signed-off-by: Wang, Yang --- demos/tests/correctness_cases.py | 12 +----------- demos/tests/run_tests.py | 4 +--- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index cd54a602126..78d8249561f 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -18,9 +18,6 @@ from cases import BASE -THREADS_NUM = os.cpu_count() - - class CorrectnessCheckerBase(ABC): def __init__(self, demo): self.filename = demo.subdirectory.replace('/', '_') + '.log' @@ -42,9 +39,6 @@ def write_to_log(self, result, test_case, device): class DemoSecurityBarrierCamera(CorrectnessCheckerBase): - def __init__(self, demo): - super().__init__(demo) - def __call__(self, output, test_case, device, execution_time=0): # Results format # {"device name": @@ -81,10 +75,6 @@ def __call__(self, output, test_case, device, execution_time=0): if case_index not in self.results[device]: self.results[device][case_index] = {} - if execution_time < 0: - self.case_index[device] += 1 - return - # Parsing the raw data output = [i.rstrip() for i in output.split('\n') if "DEBUG" in i and "ChannelId" in i] for item in output: @@ -144,7 +134,7 @@ def check_difference(self): return flag DEMOS = [ - deepcopy(BASE['security_barrier_camera_demo/cpp']) + BASE['security_barrier_camera_demo/cpp'] .update_option({'-r': None, '-n_iqs': '1'}) .add_parser(DemoSecurityBarrierCamera) ] diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index f3394c3aef4..8779ae6443e 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -356,11 +356,9 @@ def option_to_args(key, value): if not demo.check_difference(): correctness_failures += 1 print() - - print("{} failures:".format(num_failures)) if args.scope == "correctness": print("Correctness Failures: {}".format(correctness_failures)) - + print("{} failures:".format(num_failures)) for test in failed_tests: print(test) From a3e464a5fda4a75cf7b481f7c484dd58e6ec3805 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Thu, 7 Apr 2022 17:35:02 +0800 Subject: [PATCH 24/46] Update correctness checker as the common measure for all demos. Signed-off-by: Wang, Yang --- demos/tests/correctness_cases.py | 74 ++++++++++++++++---------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 78d8249561f..2285da51f0c 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -29,9 +29,43 @@ def __init__(self, demo): def __call__(self, output, test_case, device, execution_time=-1): pass - @abstractmethod def check_difference(self): - pass + flag = True + devices_list = {"AUTO:GPU,CPU" : ["CPU", "GPU"], + "AUTO:CPU" : ["CPU"], + "AUTO:GPU" : ["GPU"], + "MULTI:GPU,CPU" : ["CPU", "GPU"]} + err_msg = '' + for device in devices_list: + for target in devices_list[device]: + if device not in self.results or target not in self.results: + flag = False + err_msg += "\tMiss the results of device {} or device {}.\n".format(device, target) + if device in self.results and target in self.results: + if self.results[device] != self.results[target]: + flag = False + err_msg += "\tInconsistent results between device {} and {} \n".format(device, target) + # Show the detailed inconsistent results + for case in self.results[target]: + if self.results[device][case] != self.results[target][case]: + err_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(device, case)) + for channel in self.results[device][case]: + for frame in self.results[device][case][channel]: + if channel not in self.results[target][case] or (channel in self.results[target][case] and frame not in self.results[target][case][channel]): + err_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[device][case][channel][frame])) + err_msg += ('\t\t---------------------------------------------------------\n') + err_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(target, case)) + for channel in self.results[target][case]: + for frame in self.results[target][case][channel]: + if channel not in self.results[device][case] or (channel in self.results[device][case] and frame not in self.results[device][case][channel]): + err_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) + elif self.results[device][case][channel][frame] != self.results[target][case][channel][frame]: + err_msg += ("\t\t\tInconsist result:\n\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[target][case][channel][frame])) + err_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[device][case][channel][frame])) + err_msg += ('\t\t---------------------------------------------------------\n') + if not flag: + print("Correctness checking: Failure\n{}".format(err_msg)) + return flag def write_to_log(self, result, test_case, device): with open(self.filename, 'w') as f: @@ -40,6 +74,7 @@ def write_to_log(self, result, test_case, device): class DemoSecurityBarrierCamera(CorrectnessCheckerBase): def __call__(self, output, test_case, device, execution_time=0): + # Parsing results from raw data # Results format # {"device name": # {"case index 0": @@ -100,41 +135,8 @@ def __call__(self, output, test_case, device, execution_time=0): self.case_index[device] += 1 - def check_difference(self): - flag = True - devices_list = {"AUTO:GPU,CPU" : ["CPU", "GPU"], - "AUTO:CPU" : ["CPU"], - "AUTO:GPU" : ["GPU"], - "MULTI:GPU,CPU" : ["CPU", "GPU"]} - err_msg = '' - for device in devices_list: - for target in devices_list[device]: - if device not in self.results or target not in self.results: - flag = False - err_msg += "\tMiss the results of device {} or device {}.\n".format(device, target) - if device in self.results and target in self.results: - if self.results[device] != self.results[target]: - flag = False - err_msg += "\tInconsistent results between device {} and {} \n".format(device, target) - # Show the detailed inconsistent results - for case in self.results[target]: - if self.results[device][case] != self.results[target][case]: - err_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(device, case)) - for channel in self.results[device][case]: - for frame in self.results[device][case][channel]: - err_msg += ("\t\t\tChannel {} - Frame {} : {}\n".format(channel, frame, self.results[device][case][channel][frame])) - err_msg += ('\t\t---------------------------------------------------------\n') - err_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(target, case)) - for channel in self.results[target][case]: - for frame in self.results[target][case][channel]: - err_msg += ("\t\t\tChannel {} - Frame {} : {}\n".format(channel, frame, self.results[target][case][channel][frame])) - err_msg += ('\t\t---------------------------------------------------------\n') - if not flag: - print("Correctness checking: Failure\n{}".format(err_msg)) - return flag - DEMOS = [ - BASE['security_barrier_camera_demo/cpp'] + deepcopy(BASE['security_barrier_camera_demo/cpp']) .update_option({'-r': None, '-n_iqs': '1'}) .add_parser(DemoSecurityBarrierCamera) ] From 8631f933c0e5ba3aea34bbdfed36a99f92366d4e Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Fri, 8 Apr 2022 18:12:58 +0800 Subject: [PATCH 25/46] 1. Fix the issue that demo lost the inference of the last frame when inputting video. 2. Update the correctness checker that allows insignificant difference for the ROI. Signed-off-by: Wang, Yang --- .../security_barrier_camera_demo/cpp/main.cpp | 31 +++++++-- demos/tests/correctness_cases.py | 66 ++++++++++++++++--- 2 files changed, 81 insertions(+), 16 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index d082a41f0d8..dc88c5bede2 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -121,6 +121,8 @@ struct Context { detectorsInfers.assign(detectorInferRequests); attributesInfers.assign(attributesInferRequests); platesInfers.assign(lprInferRequests); + totalInferFrameCounter = 0; + totalFrameCount = 0; } struct { @@ -172,6 +174,11 @@ struct Context { bool isVideo; std::atomic::size_type> freeDetectionInfersCount; std::atomic frameCounter; + + // Record the inferred frames count + std::atomic totalInferFrameCounter; + std::atomic totalFrameCount; + InferRequestsContainer detectorsInfers, attributesInfers, platesInfers; PerformanceMetrics metrics; }; @@ -292,10 +299,6 @@ ReborningVideoFrame::~ReborningVideoFrame() { context.videoFramesContext.lastFrameIdsMutexes[sourceID].lock(); const auto frameId = ++context.videoFramesContext.lastframeIds[sourceID]; context.videoFramesContext.lastFrameIdsMutexes[sourceID].unlock(); - - // Stop reborning when frame ID reached to input queue size - if (!context.isVideo && frameId >= FLAGS_n_iqs) - return; std::shared_ptr reborn = std::make_shared(context, sourceID, frameId, frame); worker->push(std::make_shared(reborn)); } catch (const std::bad_weak_ptr&) {} @@ -384,7 +387,7 @@ void Drawer::process() { if (!context.isVideo) { try { // Exit only when inferences on all of frames are finished. - if (context.frameCounter >= FLAGS_n_iqs * context.readersContext.inputChannels.size()) + if (context.totalInferFrameCounter >= FLAGS_ni * context.totalFrameCount) std::shared_ptr(context.drawersContext.drawersWorker)->stop(); } catch (const std::bad_weak_ptr&) {} @@ -569,6 +572,8 @@ void DetectionsProcessor::process() { tryPush(context.detectionsProcessorsContext.detectionsProcessorsWorker, std::make_shared(sharedVideoFrame, std::move(classifiersAggregator), std::move(vehicleRects), std::move(plateRects))); } + // Count the frames passed inference + context.totalInferFrameCounter++; } bool InferTask::isReady() { @@ -591,10 +596,10 @@ void InferTask::process() { InferRequestsContainer& detectorsInfers = context.detectorsInfers; std::reference_wrapper inferRequest = detectorsInfers.inferRequests.container.back(); detectorsInfers.inferRequests.container.pop_back(); + detectorsInfers.inferRequests.mutex.unlock(); context.inferTasksContext.detector.setImage(inferRequest, sharedVideoFrame->frame); - inferRequest.get().set_callback( std::bind( [](VideoFrame::Ptr sharedVideoFrame, @@ -635,6 +640,12 @@ void Reader::process() { context.readersContext.lastCapturedFrameIds[sourceID]++; context.readersContext.lastCapturedFrameIdsMutexes[sourceID].unlock(); try { + if (context.totalInferFrameCounter < FLAGS_ni * context.totalFrameCount) + { + // Rebron this invalid frame to end the worker at next time + std::shared_ptr(context.drawersContext.drawersWorker)->push(std::make_shared(sharedVideoFrame)); + return; + } std::shared_ptr(context.drawersContext.drawersWorker)->stop(); } catch (const std::bad_weak_ptr&) {} } @@ -674,6 +685,8 @@ int main(int argc, char* argv[]) { videoCapturSourcess.push_back(std::make_shared(videoCapture, FLAGS_loop_video)); } } + + uint32_t totalFrameCount = 0; for (const std::string& file : files) { cv::Mat frame = cv::imread(file, cv::IMREAD_COLOR); if (frame.empty()) { @@ -683,8 +696,12 @@ int main(int argc, char* argv[]) { return 1; } videoCapturSourcess.push_back(std::make_shared(videoCapture, FLAGS_loop_video)); + // Get the total frame count from this video + totalFrameCount = static_cast(videoCapture.get(cv::CAP_PROP_FRAME_COUNT)); } else { imageSourcess.push_back(std::make_shared(frame, true)); + // Get the total frame count from the inputting images + totalFrameCount++; } } uint32_t channelsNum = 0 == FLAGS_ni ? videoCapturSourcess.size() + imageSourcess.size() : FLAGS_ni; @@ -802,6 +819,8 @@ int main(int argc, char* argv[]) { nireq, isVideo, nclassifiersireq, nrecognizersireq}; + // initilize the inputting frames count + context.totalFrameCount = totalFrameCount; // Create a worker after a context because the context has only weak_ptr, but the worker is going to // indirectly store ReborningVideoFrames which have a reference to the context. So there won't be a situation // when the context is destroyed and the worker still lives with its ReborningVideoFrames referring to the diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 2285da51f0c..fe405869dcf 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -29,21 +29,52 @@ def __init__(self, demo): def __call__(self, output, test_case, device, execution_time=-1): pass + def compare_roi(self, source_roi, dest_roi): + source = [] + dest = [] + if len(source_roi) != len(dest_roi): + print (source) + print (dest) + return False + for item in source_roi: + source.append(float(item)) + + for item in dest_roi: + dest.append(float(item)) + flag = True + prob_gap = 0.01 + pos_gap = 5 + for index in range(len(source)): + if index <= 1: + if abs(source[index] - dest[index]) > prob_gap: + flag = False + print (source) + print (dest) + break + else: + if abs(source[index] - dest[index]) > pos_gap: + flag = False + print (source) + print (dest) + break + return flag + def check_difference(self): flag = True - devices_list = {"AUTO:GPU,CPU" : ["CPU", "GPU"], + devices_list = { + #"AUTO:GPU,CPU" : ["CPU", "GPU"], "AUTO:CPU" : ["CPU"], - "AUTO:GPU" : ["GPU"], - "MULTI:GPU,CPU" : ["CPU", "GPU"]} + #"AUTO:GPU" : ["GPU"], + "MULTI:CPU,GPU" : ["CPU", "GPU"]} err_msg = '' + multi_correctness = {'CPU': True, 'GPU': True} for device in devices_list: for target in devices_list[device]: if device not in self.results or target not in self.results: - flag = False - err_msg += "\tMiss the results of device {} or device {}.\n".format(device, target) + flag = False + err_msg += "\tMiss the results of device {} or device {}.\n".format(device, target) if device in self.results and target in self.results: if self.results[device] != self.results[target]: - flag = False err_msg += "\tInconsistent results between device {} and {} \n".format(device, target) # Show the detailed inconsistent results for case in self.results[target]: @@ -59,10 +90,25 @@ def check_difference(self): for frame in self.results[target][case][channel]: if channel not in self.results[device][case] or (channel in self.results[device][case] and frame not in self.results[device][case][channel]): err_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) - elif self.results[device][case][channel][frame] != self.results[target][case][channel][frame]: - err_msg += ("\t\t\tInconsist result:\n\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[target][case][channel][frame])) - err_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[device][case][channel][frame])) + else: + for obj in self.results[target][case][channel][frame]: + if obj not in self.results[device][case][channel][frame]: + flag = False + err_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) + elif not self.compare_roi(self.results[device][case][channel][frame][obj],self.results[target][case][channel][frame][obj]): + if device != 'MULTI:CPU,GPU': + flag = False + else: + multi_correctness[target] = False + err_msg += ("\t\t\tInconsist result:\n\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[target][case][channel][frame])) + err_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[device][case][channel][frame])) err_msg += ('\t\t---------------------------------------------------------\n') + # Check correctness for MULTI device + for device in devices_list: + if 'MULTI:' not in device: + continue + if multi_correctness['CPU'] == False and multi_correctness['GPU'] == False: + flag = False if not flag: print("Correctness checking: Failure\n{}".format(err_msg)) return flag @@ -137,6 +183,6 @@ def __call__(self, output, test_case, device, execution_time=0): DEMOS = [ deepcopy(BASE['security_barrier_camera_demo/cpp']) - .update_option({'-r': None, '-n_iqs': '1'}) + .update_option({'-r': None,'-ni': '16', '-n_iqs': '1', '-i': '/home/wy/data_for_security_barrier_camera_demo/images_10/output.mp4'}) .add_parser(DemoSecurityBarrierCamera) ] From 91baf7dc0fab922f4608a49c36b17d9a4c234223 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Mon, 11 Apr 2022 13:22:47 +0800 Subject: [PATCH 26/46] Updata correctness checker and revert inputing images hanlder for security_barrier_camera_demo. Signed-off-by: Wang, Yang --- .../security_barrier_camera_demo/cpp/main.cpp | 4 +-- demos/tests/correctness_cases.py | 33 +++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index dc88c5bede2..54e565d2f77 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -386,9 +386,7 @@ void Drawer::process() { } else { if (!context.isVideo) { try { - // Exit only when inferences on all of frames are finished. - if (context.totalInferFrameCounter >= FLAGS_ni * context.totalFrameCount) - std::shared_ptr(context.drawersContext.drawersWorker)->stop(); + std::shared_ptr(context.drawersContext.drawersWorker)->stop(); } catch (const std::bad_weak_ptr&) {} } diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index fe405869dcf..8cd9a6bcf55 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -62,47 +62,54 @@ def compare_roi(self, source_roi, dest_roi): def check_difference(self): flag = True devices_list = { - #"AUTO:GPU,CPU" : ["CPU", "GPU"], - "AUTO:CPU" : ["CPU"], + "AUTO:GPU,CPU" : ["CPU", "GPU"], + "MULTI:CPU,GPU" : ["CPU", "GPU"], + "AUTO:CPU" : ["CPU"] #"AUTO:GPU" : ["GPU"], - "MULTI:CPU,GPU" : ["CPU", "GPU"]} + } err_msg = '' multi_correctness = {'CPU': True, 'GPU': True} for device in devices_list: + tmp_msg = '' for target in devices_list[device]: if device not in self.results or target not in self.results: flag = False err_msg += "\tMiss the results of device {} or device {}.\n".format(device, target) if device in self.results and target in self.results: + inconsist_flag = False if self.results[device] != self.results[target]: - err_msg += "\tInconsistent results between device {} and {} \n".format(device, target) + tmp_msg += "\tInconsistent results between device {} and {} \n".format(device, target) # Show the detailed inconsistent results for case in self.results[target]: if self.results[device][case] != self.results[target][case]: - err_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(device, case)) + tmp_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(device, case)) for channel in self.results[device][case]: for frame in self.results[device][case][channel]: if channel not in self.results[target][case] or (channel in self.results[target][case] and frame not in self.results[target][case][channel]): err_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[device][case][channel][frame])) - err_msg += ('\t\t---------------------------------------------------------\n') - err_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(target, case)) + tmp_msg += ('\t\t---------------------------------------------------------\n') + tmp_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(target, case)) for channel in self.results[target][case]: for frame in self.results[target][case][channel]: if channel not in self.results[device][case] or (channel in self.results[device][case] and frame not in self.results[device][case][channel]): - err_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) + tmp_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) else: for obj in self.results[target][case][channel][frame]: if obj not in self.results[device][case][channel][frame]: flag = False - err_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) + tmp_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) elif not self.compare_roi(self.results[device][case][channel][frame][obj],self.results[target][case][channel][frame][obj]): if device != 'MULTI:CPU,GPU': flag = False else: multi_correctness[target] = False - err_msg += ("\t\t\tInconsist result:\n\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[target][case][channel][frame])) - err_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[device][case][channel][frame])) - err_msg += ('\t\t---------------------------------------------------------\n') + inconsist_flag = True + tmp_msg += ("\t\t\tInconsist result:\n\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[target][case][channel][frame])) + tmp_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[device][case][channel][frame])) + tmp_msg += ('\t\t---------------------------------------------------------\n') + if inconsist_flag: + err_msg += tmp_msg + tmp_msg = '' # Check correctness for MULTI device for device in devices_list: if 'MULTI:' not in device: @@ -183,6 +190,6 @@ def __call__(self, output, test_case, device, execution_time=0): DEMOS = [ deepcopy(BASE['security_barrier_camera_demo/cpp']) - .update_option({'-r': None,'-ni': '16', '-n_iqs': '1', '-i': '/home/wy/data_for_security_barrier_camera_demo/images_10/output.mp4'}) + .update_option({'-r': None,'-ni': '16', '-n_iqs': '1', '-i': '10_images.mp4'}) .add_parser(DemoSecurityBarrierCamera) ] From 2fe3933322c3f7eb58aac8f82f963eab3b2ca0a5 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Tue, 12 Apr 2022 16:22:30 +0800 Subject: [PATCH 27/46] 1. Update correctness checker to support the multi models inputting. 2. Update the raw data format for the demo. Signed-off-by: Wang, Yang --- .../security_barrier_camera_demo/cpp/main.cpp | 21 +++++-- .../cpp/net_wrappers.hpp | 5 +- demos/tests/correctness_cases.py | 59 +++++++++++-------- 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index 54e565d2f77..877d6645609 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -227,15 +227,26 @@ class ClassifiersAggregator { std::mutex& printMutex = static_cast(sharedVideoFrame.get())->context.classifiersAggregatorPrintMutex; printMutex.lock(); if (FLAGS_r && !rawDetections.empty()) { - for (const std::string& rawDetection : rawDetections) - slog::debug << rawDetection << slog::endl; + slog::debug << "ChannelId:" << sharedVideoFrame->sourceID << "," << "FrameId:" <frameId << ","; + for (auto it = rawDetections.begin(); it != rawDetections.end(); ++it) { + if(it + 1 == rawDetections.end()) + slog::debug << *it; + else + slog::debug << *it << ","; + } // destructor assures that none uses the container + // Format: ChannleId,FrameId,ObjectId,ObjectLable,Prob,roi_x,roi_y,roi_width,roi_high,[Vehicle Attributes],[License Plate] for (const std::string& rawAttribute : rawAttributes.container) { - slog::debug << rawAttribute << slog::endl; + auto pos = rawAttribute.find(":"); + if(pos != std::string::npos) + slog::debug << "," << rawAttribute.substr(pos + 1); } for (const std::string& rawDecodedPlate : rawDecodedPlates.container) { - slog::debug << rawDecodedPlate << slog::endl; + auto pos = rawDecodedPlate.find(":"); + if(pos != std::string::npos) + slog::debug << "," << rawDecodedPlate.substr(pos + 1); } + slog::debug << slog::endl; } printMutex.unlock(); tryPush(static_cast(sharedVideoFrame.get())->context.resAggregatorsWorker, @@ -435,7 +446,7 @@ bool DetectionsProcessor::isReady() { if (requireGettingNumberOfDetections) { classifiersAggregator = std::make_shared(sharedVideoFrame); std::list results; - results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->sourceID, sharedVideoFrame->frameId, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections); + results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections); for (Detector::Result result : results) { switch (result.label) { diff --git a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp index f8810491fbc..3eadf030393 100644 --- a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp +++ b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp @@ -121,7 +121,7 @@ class Detector { } } - std::list getResults(ov::InferRequest& inferRequest, const int64_t channelID, const int64_t frameID, cv::Size upscale, std::vector& rawResults) { + std::list getResults(ov::InferRequest& inferRequest, cv::Size upscale, std::vector& rawResults) { // there is no big difference if InferReq of detector from another device is passed because the processing is the same for the same topology std::list results; ov::Tensor output_tensor = inferRequest.get_tensor(m_detectorOutputName); @@ -145,8 +145,7 @@ class Detector { rect.height = static_cast(detections[i * objectSize + 6] * upscale.height) - rect.y; results.push_back(Result{label, confidence, rect}); std::ostringstream rawResultsStream; - rawResultsStream << "ChannelId:" << channelID << ",FrameId:" << frameID << ","; - rawResultsStream << "ObjId:" << i << "," << label << "," << confidence + rawResultsStream << i << "," << label << "," << confidence << "," << rect.x << "," << rect.y << "," << rect.width << "," << rect.height; rawResults.push_back(rawResultsStream.str()); } diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 8cd9a6bcf55..a338e249cab 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -33,42 +33,49 @@ def compare_roi(self, source_roi, dest_roi): source = [] dest = [] if len(source_roi) != len(dest_roi): - print (source) - print (dest) return False + # Expected ROI format: lable, prob, x, y, w, h,...., Vehicle attribute, License plate for item in source_roi: - source.append(float(item)) + try: + source.append(float(item)) + except ValueError: + source.append(item) for item in dest_roi: - dest.append(float(item)) + try: + dest.append(float(item)) + except ValueError: + dest.append(item) flag = True - prob_gap = 0.01 - pos_gap = 5 + prob_gap = 0.1 + pos_gap = 20 for index in range(len(source)): - if index <= 1: - if abs(source[index] - dest[index]) > prob_gap: - flag = False - print (source) - print (dest) - break + if type(source[index]) == float: + if source[index] > 0 and source[index] < 1: + if abs(source[index] - dest[index]) > prob_gap: + flag = False + break + else: + if abs(source[index] - dest[index]) > pos_gap: + flag = False + break else: - if abs(source[index] - dest[index]) > pos_gap: + if source[index] != dest[index]: flag = False - print (source) - print (dest) break + return flag def check_difference(self): flag = True devices_list = { - "AUTO:GPU,CPU" : ["CPU", "GPU"], + "AUTO:CPU,GPU" : ["CPU", "GPU"], "MULTI:CPU,GPU" : ["CPU", "GPU"], - "AUTO:CPU" : ["CPU"] - #"AUTO:GPU" : ["GPU"], + "AUTO:CPU" : ["CPU"], + "AUTO:GPU" : ["GPU"], } err_msg = '' - multi_correctness = {'CPU': True, 'GPU': True} + multi_correctness = {'MULTI:CPU,GPU':{'CPU': True, 'GPU': True},'AUTO:CPU,GPU':{'CPU': True, 'GPU': True}} for device in devices_list: tmp_msg = '' for target in devices_list[device]: @@ -99,22 +106,25 @@ def check_difference(self): flag = False tmp_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) elif not self.compare_roi(self.results[device][case][channel][frame][obj],self.results[target][case][channel][frame][obj]): - if device != 'MULTI:CPU,GPU': + if device != 'MULTI:CPU,GPU' and device != 'AUTO:CPU,GPU': flag = False else: - multi_correctness[target] = False + multi_correctness[device][target] = False inconsist_flag = True tmp_msg += ("\t\t\tInconsist result:\n\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[target][case][channel][frame])) tmp_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[device][case][channel][frame])) tmp_msg += ('\t\t---------------------------------------------------------\n') + if not inconsist_flag: + tmp_msg = '' if inconsist_flag: err_msg += tmp_msg tmp_msg = '' + inconsist_flag = False # Check correctness for MULTI device for device in devices_list: - if 'MULTI:' not in device: + if 'MULTI:CPU,GPU' != device and 'AUTO:CPU,GPU' != device: continue - if multi_correctness['CPU'] == False and multi_correctness['GPU'] == False: + if multi_correctness[device]['CPU'] == False and multi_correctness[device]['GPU'] == False: flag = False if not flag: print("Correctness checking: Failure\n{}".format(err_msg)) @@ -151,7 +161,6 @@ def __call__(self, output, test_case, device, execution_time=0): # ..... # } # } - # Generate case id for each device if device not in self.case_index: self.case_index[device] = 0 @@ -181,7 +190,7 @@ def __call__(self, output, test_case, device, execution_time=0): # Object ID label_prob_pos_results = [] - objid = item[2].split(':')[1] + objid = item[2] if objid not in self.results[device][case_index][channel][frame]: self.results[device][case_index][channel][frame][objid] = label_prob_pos_results self.results[device][case_index][channel][frame][objid] = item[3:] From 0914845c382fdf1d6075d0c0e874268590393d9c Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 13 Apr 2022 14:13:10 +0800 Subject: [PATCH 28/46] Update exit code when correctness checking falied. Signed-off-by: Wang, Yang --- demos/tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index 3a38bd2f896..6e8070174b3 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -362,7 +362,7 @@ def option_to_args(key, value): for test in failed_tests: print(test) - sys.exit(0 if num_failures == 0 else 1) + sys.exit(0 if num_failures == 0 and correctness_failures == 0 else 1) if __name__ == '__main__': From 796e315ef40fa20de09096352abb6f25137361d5 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Fri, 15 Apr 2022 13:38:24 +0800 Subject: [PATCH 29/46] Modify the input dataset path when updating option '-i' for demo. Signed-off-by: Wang, Yang --- demos/tests/cases.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/demos/tests/cases.py b/demos/tests/cases.py index f4c1e7fab2f..99d75b6cb6e 100644 --- a/demos/tests/cases.py +++ b/demos/tests/cases.py @@ -63,7 +63,11 @@ def update_case(self, case, updated_options, with_replacement=False): if not updated_options: return new_options = case.options.copy() for key, value in updated_options.items(): - new_options[key] = value + # Modify path to the new dataset if updating option '-i' + if key == '-i': + new_options[key] = TestDataArg(value) + else: + new_options[key] = value new_case = case._replace(options=new_options) if with_replacement: self.test_cases.remove(case) From f33983d72d0e08156c9859e00b1f72f557d907e2 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Fri, 15 Apr 2022 16:37:23 +0800 Subject: [PATCH 30/46] Update correctness checker. Signed-off-by: Wang, Yang --- demos/tests/correctness_cases.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index a338e249cab..051588884e1 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -69,13 +69,14 @@ def compare_roi(self, source_roi, dest_roi): def check_difference(self): flag = True devices_list = { - "AUTO:CPU,GPU" : ["CPU", "GPU"], - "MULTI:CPU,GPU" : ["CPU", "GPU"], + "AUTO:GPU,CPU" : ["CPU", "GPU"], + "MULTI:GPU,CPU" : ["CPU", "GPU"], "AUTO:CPU" : ["CPU"], "AUTO:GPU" : ["GPU"], } err_msg = '' - multi_correctness = {'MULTI:CPU,GPU':{'CPU': True, 'GPU': True},'AUTO:CPU,GPU':{'CPU': True, 'GPU': True}} + multi_correctness = {'MULTI:GPU,CPU':{'CPU': True, 'GPU': True},'AUTO:GPU,CPU':{'CPU': True, 'GPU': True}} + multi_inconsist_results = {} for device in devices_list: tmp_msg = '' for target in devices_list[device]: @@ -106,7 +107,7 @@ def check_difference(self): flag = False tmp_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) elif not self.compare_roi(self.results[device][case][channel][frame][obj],self.results[target][case][channel][frame][obj]): - if device != 'MULTI:CPU,GPU' and device != 'AUTO:CPU,GPU': + if device != 'MULTI:GPU,CPU' and device != 'AUTO:GPU,CPU': flag = False else: multi_correctness[device][target] = False @@ -117,16 +118,26 @@ def check_difference(self): if not inconsist_flag: tmp_msg = '' if inconsist_flag: - err_msg += tmp_msg + if device == 'MULTI:GPU,CPU' or device == 'AUTO:GPU,CPU': + if device not in multi_inconsist_results: + multi_inconsist_results[device] = '' + multi_inconsist_results[device] += tmp_msg + else: + err_msg += tmp_msg tmp_msg = '' inconsist_flag = False # Check correctness for MULTI device for device in devices_list: - if 'MULTI:CPU,GPU' != device and 'AUTO:CPU,GPU' != device: + if 'MULTI:GPU,CPU' != device and 'AUTO:GPU,CPU' != device: continue if multi_correctness[device]['CPU'] == False and multi_correctness[device]['GPU'] == False: flag = False if not flag: + multi_msg = '' + for device in multi_inconsist_results: + if multi_correctness[device]['CPU'] == False and multi_correctness[device]['GPU'] == False: + multi_msg += multi_inconsist_results[device] + err_msg += multi_msg print("Correctness checking: Failure\n{}".format(err_msg)) return flag @@ -202,3 +213,4 @@ def __call__(self, output, test_case, device, execution_time=0): .update_option({'-r': None,'-ni': '16', '-n_iqs': '1', '-i': '10_images.mp4'}) .add_parser(DemoSecurityBarrierCamera) ] + From 7c47ce79e935140eb5cc697d0dc0581af9e981fd Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Sun, 24 Apr 2022 14:08:07 +0800 Subject: [PATCH 31/46] Correct the output layer order of the attributes model for the security_barrier_camera demo. Signed-off-by: Wang, Yang --- demos/security_barrier_camera_demo/cpp/main.cpp | 1 - demos/security_barrier_camera_demo/cpp/net_wrappers.hpp | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index 877d6645609..bb67850afae 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -754,7 +754,6 @@ int main(int argc, char* argv[]) { } core.set_property("CPU", ov::affinity(ov::Affinity::NONE)); core.set_property("CPU", ov::streams::num((device_nstreams.count("CPU") > 0 ? ov::streams::Num(device_nstreams["CPU"]) : ov::streams::AUTO))); - device_nstreams["CPU"] = core.get_property("CPU", ov::streams::num); } diff --git a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp index 3eadf030393..48df4c5c90d 100644 --- a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp +++ b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp @@ -186,10 +186,10 @@ class VehicleAttributesClassifier { throw std::logic_error("Vehicle Attribs Network expects networks having two outputs"); } - // color is the first output - m_outputNameForColor = outputs[0].get_any_name(); - // type is the second output. - m_outputNameForType = outputs[1].get_any_name(); + // type is the first output + m_outputNameForType = outputs[0].get_any_name(); + // color is the second output. + m_outputNameForColor = outputs[1].get_any_name(); ov::preprocess::PrePostProcessor ppp(model); From 2f066d1390e0e6d94df5f154bcebbd8929f77fd2 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Tue, 26 Apr 2022 09:20:42 +0800 Subject: [PATCH 32/46] 1. Stop reborning if images frame ID is invalid. 2. clone image frame to avoid changing. Signed-off-by: Wang, Yang --- demos/common/cpp/utils/include/utils/input_wrappers.hpp | 6 ++++-- demos/security_barrier_camera_demo/cpp/main.cpp | 3 +++ demos/tests/correctness_cases.py | 5 +++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/demos/common/cpp/utils/include/utils/input_wrappers.hpp b/demos/common/cpp/utils/include/utils/input_wrappers.hpp index eff38a72771..a455177385b 100644 --- a/demos/common/cpp/utils/include/utils/input_wrappers.hpp +++ b/demos/common/cpp/utils/include/utils/input_wrappers.hpp @@ -126,11 +126,13 @@ class ImageSource: public IInputSource { return false; } else { subscribedInputChannels.erase(subscribedInputChannelsIt); - mat = im; + // clone to avoid that the image is shared and changed. + mat = im.clone(); return true; } } else { - mat = im; + // clone to avoid that the image is shared and changed. + mat = im.clone(); return true; } } diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index bb67850afae..7731300767a 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -310,6 +310,9 @@ ReborningVideoFrame::~ReborningVideoFrame() { context.videoFramesContext.lastFrameIdsMutexes[sourceID].lock(); const auto frameId = ++context.videoFramesContext.lastframeIds[sourceID]; context.videoFramesContext.lastFrameIdsMutexes[sourceID].unlock(); + // Stop reborning if image frameId is invalid + if(!context.isVideo && frameId >= FLAGS_n_iqs) + return; std::shared_ptr reborn = std::make_shared(context, sourceID, frameId, frame); worker->push(std::make_shared(reborn)); } catch (const std::bad_weak_ptr&) {} diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 051588884e1..65196a105ee 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -113,6 +113,11 @@ def check_difference(self): multi_correctness[device][target] = False inconsist_flag = True tmp_msg += ("\t\t\tInconsist result:\n\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[target][case][channel][frame])) + if target == 'CPU': + tmp_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format('GPU', channel, frame, self.results['GPU'][case][channel][frame])) + else: + tmp_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format('CPU', channel, frame, self.results['CPU'][case][channel][frame])) + tmp_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[device][case][channel][frame])) tmp_msg += ('\t\t---------------------------------------------------------\n') if not inconsist_flag: From 1562b9b16a1f8c8954637e54ac3b95fb767383c5 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Thu, 28 Apr 2022 20:35:58 +0800 Subject: [PATCH 33/46] Update correctness checking logic. Signed-off-by: Wang, Yang --- .../security_barrier_camera_demo/cpp/main.cpp | 28 +- demos/tests/correctness_cases.py | 251 ++++++++++++------ 2 files changed, 190 insertions(+), 89 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index 7731300767a..f02b5f013c9 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -229,22 +229,30 @@ class ClassifiersAggregator { if (FLAGS_r && !rawDetections.empty()) { slog::debug << "ChannelId:" << sharedVideoFrame->sourceID << "," << "FrameId:" <frameId << ","; for (auto it = rawDetections.begin(); it != rawDetections.end(); ++it) { - if(it + 1 == rawDetections.end()) - slog::debug << *it; + if(it == std::prev(rawDetections.end())) + slog::debug << *it << "\t"; else slog::debug << *it << ","; } // destructor assures that none uses the container // Format: ChannleId,FrameId,ObjectId,ObjectLable,Prob,roi_x,roi_y,roi_width,roi_high,[Vehicle Attributes],[License Plate] - for (const std::string& rawAttribute : rawAttributes.container) { - auto pos = rawAttribute.find(":"); - if(pos != std::string::npos) - slog::debug << "," << rawAttribute.substr(pos + 1); + for (auto it = rawAttributes.container.begin(); it != rawAttributes.container.end(); ++it) { + auto pos = it->find(":"); + if(pos != std::string::npos) { + if(it == std::prev(rawAttributes.container.end())) + slog::debug << it->substr(pos + 1) << "\t"; + else + slog::debug << it->substr(pos + 1) << ","; + } } - for (const std::string& rawDecodedPlate : rawDecodedPlates.container) { - auto pos = rawDecodedPlate.find(":"); - if(pos != std::string::npos) - slog::debug << "," << rawDecodedPlate.substr(pos + 1); + for (auto it = rawDecodedPlates.container.begin(); it != rawDecodedPlates.container.end(); ++it) { + auto pos = it->find(":"); + if(pos != std::string::npos) { + if(it == std::prev(rawDecodedPlates.container.end())) + slog::debug << it->substr(pos + 1); + else + slog::debug << it->substr(pos + 1) << ","; + } } slog::debug << slog::endl; } diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 65196a105ee..6dd2578ee21 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -34,18 +34,13 @@ def compare_roi(self, source_roi, dest_roi): dest = [] if len(source_roi) != len(dest_roi): return False - # Expected ROI format: lable, prob, x, y, w, h,...., Vehicle attribute, License plate + # Expected ROI format: able, prob, x, y, w, h,.... for item in source_roi: - try: - source.append(float(item)) - except ValueError: - source.append(item) + source.append(float(item)) for item in dest_roi: - try: - dest.append(float(item)) - except ValueError: - dest.append(item) + dest.append(float(item)) + flag = True prob_gap = 0.1 pos_gap = 20 @@ -74,77 +69,168 @@ def check_difference(self): "AUTO:CPU" : ["CPU"], "AUTO:GPU" : ["GPU"], } - err_msg = '' - multi_correctness = {'MULTI:GPU,CPU':{'CPU': True, 'GPU': True},'AUTO:GPU,CPU':{'CPU': True, 'GPU': True}} - multi_inconsist_results = {} + # { + # device 0: { target 0: {case 0: True/False, case 1: True/False, ...},...,target n:{} }, + # device 1: { target 1: {case 0: True/False, case 1: True/False, ...},...,target n:{} }, + # } + multi_correctness = {} + + # For multi devices like AUTO:GPU,CPU, MULTI:GPU,CPU + #{ + # device 0:{ + # case 0: { + # channel 0: { + # frame 0: [roi_result, attribute_result, license_plate_results], + # frame 1: [roi_result, attribute_result, license_plate_results], + # ... + # }, + # channel 1: {}, + # ......... + # }, + # case 1:{}, + # ....... + # } + # device 1:{}, + #} + multi_correctness_devices = {} + + #{ + # device 0: {case 0: "error msg", case 1: "error msg", ...}, + # device 1: {case 0: "error msg", case 1: "error msg", ...}, + # .... + # } + multi_correctness_errmsg = {} + for device in devices_list: + multi_correctness[device] = {} + multi_correctness_errmsg[device] = {} + if 'GPU' in device and 'CPU' in device: + multi_correctness_devices[device] = {} + for target in devices_list[device]: + multi_correctness[device][target] = {} + if 'GPU' in device and 'CPU' in device: + multi_correctness_devices[device][target] = {} for device in devices_list: - tmp_msg = '' for target in devices_list[device]: if device not in self.results or target not in self.results: - flag = False - err_msg += "\tMiss the results of device {} or device {}.\n".format(device, target) - if device in self.results and target in self.results: - inconsist_flag = False - if self.results[device] != self.results[target]: - tmp_msg += "\tInconsistent results between device {} and {} \n".format(device, target) - # Show the detailed inconsistent results - for case in self.results[target]: - if self.results[device][case] != self.results[target][case]: - tmp_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(device, case)) - for channel in self.results[device][case]: - for frame in self.results[device][case][channel]: - if channel not in self.results[target][case] or (channel in self.results[target][case] and frame not in self.results[target][case][channel]): - err_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[device][case][channel][frame])) - tmp_msg += ('\t\t---------------------------------------------------------\n') - tmp_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(target, case)) - for channel in self.results[target][case]: - for frame in self.results[target][case][channel]: - if channel not in self.results[device][case] or (channel in self.results[device][case] and frame not in self.results[device][case][channel]): - tmp_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) - else: - for obj in self.results[target][case][channel][frame]: - if obj not in self.results[device][case][channel][frame]: - flag = False - tmp_msg += ("\t\t\t[Not Found on {}]Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[target][case][channel][frame])) - elif not self.compare_roi(self.results[device][case][channel][frame][obj],self.results[target][case][channel][frame][obj]): - if device != 'MULTI:GPU,CPU' and device != 'AUTO:GPU,CPU': - flag = False - else: - multi_correctness[device][target] = False - inconsist_flag = True - tmp_msg += ("\t\t\tInconsist result:\n\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(target, channel, frame, self.results[target][case][channel][frame])) - if target == 'CPU': - tmp_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format('GPU', channel, frame, self.results['GPU'][case][channel][frame])) - else: - tmp_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format('CPU', channel, frame, self.results['CPU'][case][channel][frame])) - - tmp_msg += ("\t\t\t\t[{}] Channel {} - Frame {} : {}\n".format(device, channel, frame, self.results[device][case][channel][frame])) - tmp_msg += ('\t\t---------------------------------------------------------\n') - if not inconsist_flag: - tmp_msg = '' - if inconsist_flag: - if device == 'MULTI:GPU,CPU' or device == 'AUTO:GPU,CPU': - if device not in multi_inconsist_results: - multi_inconsist_results[device] = '' - multi_inconsist_results[device] += tmp_msg - else: - err_msg += tmp_msg - tmp_msg = '' - inconsist_flag = False - # Check correctness for MULTI device + multi_correctness[device][target] = {} + multi_correctness_errmsg[device]['-1'] = "\tMiss the results of device {} or device {}.\n".format(device, target) + continue + for case in self.results[target]: + if case not in multi_correctness[device][target]: + multi_correctness[device][target][case] = True + if case not in multi_correctness_errmsg[device]: + multi_correctness_errmsg[device][case] = '' + for channel in self.results[device][case]: + for frame in self.results[device][case][channel]: + if channel not in self.results[target][case] or (channel in self.results[target][case] and frame not in self.results[target][case][channel]): + multi_correctness[device][target][case] = False + multi_correctness_errmsg[device][case] += "[Device: {}- Case: {}][Not Found on {}]Channel {} - Frame {} : {}\n".format(device, case, target, channel, frame, self.results[device][case][channel][frame]) + else: + for obj in self.results[target][case][channel][frame]: + if obj not in self.results[device][case][channel][frame]: + multi_correctness[device][target][case] = False + multi_correctness_errmsg[device][case] += "[Device: {}- Case: {}][Not Found on {}]Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[target][case][channel][frame]) + else: + if 'CPU' in device and 'GPU' in device: + if case not in multi_correctness_devices[device][target]: + multi_correctness_devices[device][target][case] = {} + if channel not in multi_correctness_devices[device][target][case]: + multi_correctness_devices[device][target][case][channel] = {} + if frame not in multi_correctness_devices[device][target][case][channel]: + multi_correctness_devices[device][target][case][channel][frame] = [] + + for i in range(len(self.results[device][case][channel][frame][obj])): + if i == 0: + # Compared ROI + device_vehicle_roi = self.results[device][case][channel][frame][obj][i] + target_vehicle_roi = self.results[target][case][channel][frame][obj][i] + flag_roi = self.compare_roi(device_vehicle_roi, target_vehicle_roi) + if 'CPU' in device and 'GPU' in device: + multi_correctness_devices[device][target][case][channel][frame].append(flag_roi) + else: + if not flag_roi: + multi_correctness[device][target][case] = False + tmp_msg = ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[device][case][channel][frame])) + tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, target, channel, frame, self.results[target][case][channel][frame])) + multi_correctness_errmsg[device][case] += tmp_msg + else: + # Compare attribute/license plate + device_vehicle_attr = self.results[device][case][channel][frame][obj][i] + target_vehicle_attr = self.results[target][case][channel][frame][obj][i] + if 'CPU' in device and 'GPU' in device: + multi_correctness_devices[device][target][case][channel][frame].append(device_vehicle_attr == target_vehicle_attr) + else: + if device_vehicle_attr != target_vehicle_attr: + multi_correctness[device][target][case] = False + if 'CPU' in device and 'GPU' in device: + # Check if correctness result between device and target for multi devices + consistent_flag = False + for flag in multi_correctness_devices[device][target][case][channel][frame]: + if flag == True: + consistent_flag = True + break + if not consistent_flag: + tmp_msg = ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, target, channel, frame, self.results[target][case][channel][frame])) + if 'CPU' in device and 'GPU' in device and target == 'CPU': + tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, 'GPU', channel, frame, self.results['GPU'][case][channel][frame])) + elif 'CPU' in device and 'GPU' in device and target == 'GPU': + tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, 'CPU', channel, frame, self.results['CPU'][case][channel][frame])) + tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[device][case][channel][frame])) + multi_correctness_errmsg[device][case] += tmp_msg + + final_correctness_flag = True for device in devices_list: if 'MULTI:GPU,CPU' != device and 'AUTO:GPU,CPU' != device: - continue - if multi_correctness[device]['CPU'] == False and multi_correctness[device]['GPU'] == False: - flag = False - if not flag: - multi_msg = '' - for device in multi_inconsist_results: - if multi_correctness[device]['CPU'] == False and multi_correctness[device]['GPU'] == False: - multi_msg += multi_inconsist_results[device] - err_msg += multi_msg - print("Correctness checking: Failure\n{}".format(err_msg)) - return flag + for target in multi_correctness[device]: + consistent_flag = True + err_msg = '' + if len(multi_correctness[device][target]) == 0: + final_correctness_flag = False + consistent_flag = False + # Miss results for device + err_msg += multi_correctness_errmsg[device]['-1'] + else: + for case in multi_correctness[device][target]: + flag = multi_correctness[device][target][case] + if flag == False: + final_correctness_flag = False + consistent_flag = False + err_msg += multi_correctness_errmsg[device][case] + if not consistent_flag: + print("Checking correctness between {} and {} : Failure.\n{}".format(device, devices_list[device], err_msg)) + else: + print("Checking correctness between {} and {} : PASS.\n".format(device, devices_list[device])) + else: + consistent_flag = True + err_msg = '' + for case in multi_correctness[device]['CPU']: + if multi_correctness[device]['CPU'][case] == False: + final_correctness_flag = False + consistent_flag = False + err_msg += multi_correctness_errmsg[device][case] + else: + for channel in multi_correctness_devices[device]['GPU'][case]: + for frame in multi_correctness_devices[device]['GPU'][case][channel]: + frame_flag = True + for i in range(len(multi_correctness_devices[device]['GPU'][case][channel][frame])): + gpu_frame_consistent_flag = multi_correctness_devices[device]['GPU'][case][channel][frame][i] + cpu_frame_consistent_flag = multi_correctness_devices[device]['CPU'][case][channel][frame][i] + if gpu_frame_consistent_flag == False and cpu_frame_consistent_flag == False: + frame_flag = False + if not frame_flag: + final_correctness_flag = False + consistent_flag = False + err_msg += "\tDevice: {} - case: {} - channel: {} - frame: {}: {}\n".format(device, case, channel, frame, self.results[device][case][channel][frame][obj]) + for target in devices_list[device]: + err_msg += "\tDevice: {} - case: {} - channel: {} - frame: {}: {}\n".format(target, case, channel, frame, self.results[device][case][channel][frame][obj]) + for target in devices_list[device]: + err_msg += "\tDevice : {} - case: {} - channel: {} - frame: {}: {}\n".format(target, case, channel, frame, multi_correctness_devices[device]['GPU'][case][channel][frame]) + + if not consistent_flag: + print("Checking correctness between {} and {} : Failure.\n{}".format(device, devices_list[device], err_msg)) + else: + print("Checking correctness between {} and {} : PASS.\n".format(device, devices_list[device])) + return final_correctness_flag def write_to_log(self, result, test_case, device): with open(self.filename, 'w') as f: @@ -159,10 +245,10 @@ def __call__(self, output, test_case, device, execution_time=0): # {"case index 0": # {"channel id 0": # {"frame id 0": - # {"object id 0":{"label:xx,prob:xx,x,y,width,hight"}, - # {"object id 1":{"label:xx,prob:xx,x,y,width,hight"}, + # {"object id 0":[["label", "prob", "x", "y", "width", "hight"],"Vehicle attribute", "License plate"], + # {"object id 1":["label", "prob", "x", "y", "width", "hight","Vehicle attribute", "License plate"], # ..................... - # {"object id n":{"label:xx,prob:xx,x,y,width,hight"} + # {"object id n":["label", "prob", "x", "y", "width", "hight","Vehicle attribute", "License plate"], # }, # ..... # {"frame id n": @@ -209,7 +295,14 @@ def __call__(self, output, test_case, device, execution_time=0): objid = item[2] if objid not in self.results[device][case_index][channel][frame]: self.results[device][case_index][channel][frame][objid] = label_prob_pos_results - self.results[device][case_index][channel][frame][objid] = item[3:] + #self.results[device][case_index][channel][frame][objid] = item[3:] + roi_attr_license = ",".join(item[3:]).split('\t') + for index in range(len(roi_attr_license)): + if index == 0: + item = roi_attr_license[index].split(',') + self.results[device][case_index][channel][frame][objid].append(item) + else: + self.results[device][case_index][channel][frame][objid].append(roi_attr_license[index]) self.case_index[device] += 1 From d2d37e322abada743b86c3790fa87bd2c7d46853 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Thu, 5 May 2022 15:29:50 +0800 Subject: [PATCH 34/46] 1. fix the bug in the security demo that lost the results of the inferred frames. 2. update correctness checking logic that could validate the correctness the multi results within 1 frame individually. Signed-off-by: Wang, Yang --- .../security_barrier_camera_demo/cpp/main.cpp | 9 +- .../cpp/net_wrappers.hpp | 8 ++ demos/tests/correctness_cases.py | 104 +++++++++++++----- 3 files changed, 90 insertions(+), 31 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index f02b5f013c9..3125ecb9198 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -519,6 +519,8 @@ void DetectionsProcessor::process() { const cv::Rect vehicleRect = *vehicleRectsIt; ov::InferRequest& attributesRequest = *attributesRequestIt; context.detectionsProcessorsContext.vehicleAttributesClassifier.setImage(attributesRequest, sharedVideoFrame->frame, vehicleRect); + // Decrease total inferred frames count by 1 when the ROI of frame has availiable attributes. + context.totalInferFrameCounter--; attributesRequest.set_callback( std::bind( @@ -538,6 +540,8 @@ void DetectionsProcessor::process() { classifiersAggregator->push( BboxAndDescr{BboxAndDescr::ObjectType::VEHICLE, rect, attributes.first + ' ' + attributes.second}); context.attributesInfers.inferRequests.lockedPushBack(attributesRequest); + // Increased the total inferred frames count by 1 when attributes classification is done. + context.totalInferFrameCounter++; }, classifiersAggregator, std::ref(attributesRequest), vehicleRect, @@ -558,7 +562,8 @@ void DetectionsProcessor::process() { const cv::Rect plateRect = *plateRectsIt; ov::InferRequest& lprRequest = *lprRequestsIt; context.detectionsProcessorsContext.lpr.setImage(lprRequest, sharedVideoFrame->frame, plateRect); - + // Decrease the total inferred frames count by 1 when the ROI of frame has license plate. + context.totalInferFrameCounter--; lprRequest.set_callback( std::bind( [](std::shared_ptr classifiersAggregator, @@ -574,6 +579,8 @@ void DetectionsProcessor::process() { } classifiersAggregator->push(BboxAndDescr{BboxAndDescr::ObjectType::PLATE, rect, std::move(result)}); context.platesInfers.inferRequests.lockedPushBack(lprRequest); + // Increased by 1 total inferred frames count by 1 when license plate recognization is done. + context.totalInferFrameCounter++; }, classifiersAggregator, std::ref(lprRequest), plateRect, diff --git a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp index 48df4c5c90d..ccb9a181271 100644 --- a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp +++ b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp @@ -140,9 +140,17 @@ class Detector { cv::Rect rect; rect.x = static_cast(detections[i * objectSize + 3] * upscale.width); + if(rect.x < 0) + rect.x = 0; rect.y = static_cast(detections[i * objectSize + 4] * upscale.height); + if(rect.y < 0) + rect.y = 0; rect.width = static_cast(detections[i * objectSize + 5] * upscale.width) - rect.x; + if(rect.width > upscale.width) + rect.width = upscale.width; rect.height = static_cast(detections[i * objectSize + 6] * upscale.height) - rect.y; + if(rect.height > upscale.height) + rect.height = upscale.height; results.push_back(Result{label, confidence, rect}); std::ostringstream rawResultsStream; rawResultsStream << i << "," << label << "," << confidence diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 6dd2578ee21..8e047165380 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -69,13 +69,14 @@ def check_difference(self): "AUTO:CPU" : ["CPU"], "AUTO:GPU" : ["GPU"], } + # Record the results for the single device inputting like AUTO:CPU, AUTO:GPU # { # device 0: { target 0: {case 0: True/False, case 1: True/False, ...},...,target n:{} }, # device 1: { target 1: {case 0: True/False, case 1: True/False, ...},...,target n:{} }, # } multi_correctness = {} - # For multi devices like AUTO:GPU,CPU, MULTI:GPU,CPU + # Record the results for the multi devices inputting like AUTO:GPU,CPU, MULTI:GPU,CPU #{ # device 0:{ # case 0: { @@ -94,7 +95,8 @@ def check_difference(self): #} multi_correctness_devices = {} - #{ + # Record the detailed msg for the inconsistent results + # { # device 0: {case 0: "error msg", case 1: "error msg", ...}, # device 1: {case 0: "error msg", case 1: "error msg", ...}, # .... @@ -115,11 +117,15 @@ def check_difference(self): multi_correctness[device][target] = {} multi_correctness_errmsg[device]['-1'] = "\tMiss the results of device {} or device {}.\n".format(device, target) continue + #if self.results[device] != self.results[target]: + # # Show the detailed inconsistent results for case in self.results[target]: if case not in multi_correctness[device][target]: multi_correctness[device][target][case] = True if case not in multi_correctness_errmsg[device]: multi_correctness_errmsg[device][case] = '' + + #if self.results[device][case] != self.results[target][case]: for channel in self.results[device][case]: for frame in self.results[device][case][channel]: if channel not in self.results[target][case] or (channel in self.results[target][case] and frame not in self.results[target][case][channel]): @@ -131,6 +137,15 @@ def check_difference(self): multi_correctness[device][target][case] = False multi_correctness_errmsg[device][case] += "[Device: {}- Case: {}][Not Found on {}]Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[target][case][channel][frame]) else: + #print("======Case:{} - Device: {} - Target: {} - channel: {} - frame: {}=======".format(case, device, target, channel, frame)) + #print("Device {}: {}".format(device, self.results[device][case][channel][frame][obj])) + #print("{}: {}".format(target, self.results[target][case][channel][frame][obj])) + #if 'CPU' in device and 'GPU' in device: + # if target == 'CPU': + # print("GPU: {}".format(self.results['GPU'][case][channel][frame][obj])) + # elif target == 'GPU': + # print("CPU: {}".format(self.results['CPU'][case][channel][frame][obj])) + if 'CPU' in device and 'GPU' in device: if case not in multi_correctness_devices[device][target]: multi_correctness_devices[device][target][case] = {} @@ -162,6 +177,14 @@ def check_difference(self): else: if device_vehicle_attr != target_vehicle_attr: multi_correctness[device][target][case] = False + tmp_msg = ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[device][case][channel][frame])) + tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, target, channel, frame, self.results[target][case][channel][frame])) + multi_correctness_errmsg[device][case] += tmp_msg + #if 'CPU' in device and 'GPU' in device: + # print(multi_correctness_devices[device][target][case][channel][frame]) + #else: + # print(multi_correctness[device][target][case]) + #print("======================================") if 'CPU' in device and 'GPU' in device: # Check if correctness result between device and target for multi devices consistent_flag = False @@ -178,6 +201,18 @@ def check_difference(self): tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[device][case][channel][frame])) multi_correctness_errmsg[device][case] += tmp_msg + #print("=====================================") + #print(multi_correctness) + #print("=====================================") + print(multi_correctness_errmsg) + #print("=====================================") + #for device in multi_correctness_devices: + # for target in multi_correctness_devices[device]: + # for case in multi_correctness_devices[device][target]: + # for channel in multi_correctness_devices[device][target][case]: + # print("===== Device:{} - target: {} - Case: {} - channel: {}=====".format(device, target, case, channel)) + # print(multi_correctness_devices[device][target][case][channel]) + #print("=====================================") final_correctness_flag = True for device in devices_list: if 'MULTI:GPU,CPU' != device and 'AUTO:GPU,CPU' != device: @@ -197,39 +232,48 @@ def check_difference(self): consistent_flag = False err_msg += multi_correctness_errmsg[device][case] if not consistent_flag: - print("Checking correctness between {} and {} : Failure.\n{}".format(device, devices_list[device], err_msg)) + print("Checking device: {} - target : {} - : Fail.\n{}".format(device, devices_list[device], err_msg)) else: - print("Checking correctness between {} and {} : PASS.\n".format(device, devices_list[device])) + print("Checking device: {} - target : {} - : PASS.\n".format(device, devices_list[device])) else: consistent_flag = True err_msg = '' - for case in multi_correctness[device]['CPU']: - if multi_correctness[device]['CPU'][case] == False: - final_correctness_flag = False - consistent_flag = False - err_msg += multi_correctness_errmsg[device][case] - else: - for channel in multi_correctness_devices[device]['GPU'][case]: - for frame in multi_correctness_devices[device]['GPU'][case][channel]: - frame_flag = True - for i in range(len(multi_correctness_devices[device]['GPU'][case][channel][frame])): - gpu_frame_consistent_flag = multi_correctness_devices[device]['GPU'][case][channel][frame][i] - cpu_frame_consistent_flag = multi_correctness_devices[device]['CPU'][case][channel][frame][i] - if gpu_frame_consistent_flag == False and cpu_frame_consistent_flag == False: - frame_flag = False - if not frame_flag: - final_correctness_flag = False - consistent_flag = False - err_msg += "\tDevice: {} - case: {} - channel: {} - frame: {}: {}\n".format(device, case, channel, frame, self.results[device][case][channel][frame][obj]) - for target in devices_list[device]: - err_msg += "\tDevice: {} - case: {} - channel: {} - frame: {}: {}\n".format(target, case, channel, frame, self.results[device][case][channel][frame][obj]) - for target in devices_list[device]: - err_msg += "\tDevice : {} - case: {} - channel: {} - frame: {}: {}\n".format(target, case, channel, frame, multi_correctness_devices[device]['GPU'][case][channel][frame]) + if len(multi_correctness[device][target]) == 0: + final_correctness_flag = False + consistent_flag = False + # Miss results for device + err_msg += multi_correctness_errmsg[device]['-1'] + else: + for case in multi_correctness[device]['CPU']: + if multi_correctness[device]['CPU'][case] == False: + final_correctness_flag = False + consistent_flag = False + err_msg += multi_correctness_errmsg[device][case] + #print(multi_correctness_errmsg[device][target][case]) + else: + for channel in multi_correctness_devices[device]['GPU'][case]: + for frame in multi_correctness_devices[device]['GPU'][case][channel]: + frame_flag = True + for i in range(len(multi_correctness_devices[device]['GPU'][case][channel][frame])): + gpu_frame_consistent_flag = multi_correctness_devices[device]['GPU'][case][channel][frame][i] + cpu_frame_consistent_flag = multi_correctness_devices[device]['CPU'][case][channel][frame][i] + if gpu_frame_consistent_flag == False and cpu_frame_consistent_flag == False: + frame_flag = False + #print("checking result: {}\nDevice : GPU - case: {} - channel: {} - frame: {}: {}".format(frame_flag, case, channel, frame, multi_correctness_devices[device]['GPU'][case][channel][frame])) + #print("Device : CPU - case: {} - channel: {} - frame: {}: {}".format(case, channel, frame, multi_correctness_devices[device]['CPU'][case][channel][frame])) + if not frame_flag: + final_correctness_flag = False + consistent_flag = False + err_msg += "Inconsistent result:\n\tDevice: {} - case: {} - channel: {} - frame: {}: {}\n".format(device, case, channel, frame, self.results[device][case][channel][frame]) + for target in devices_list[device]: + err_msg += "\tDevice: {} - case: {} - channel: {} - frame: {}: {}\n".format(target, case, channel, frame, self.results[target][case][channel][frame]) + for target in devices_list[device]: + err_msg += "\tDevice: {} - case: {} - channel: {} - frame: {}: {}\n".format(target, case, channel, frame, multi_correctness_devices[device][target][case][channel][frame]) if not consistent_flag: - print("Checking correctness between {} and {} : Failure.\n{}".format(device, devices_list[device], err_msg)) + print("Checking device: {} - target : {} - : Fail.\n{}".format(device, devices_list[device], err_msg)) else: - print("Checking correctness between {} and {} : PASS.\n".format(device, devices_list[device])) + print("Checking device: {} - target : {} - : PASS.\n".format(device, devices_list[device])) return final_correctness_flag def write_to_log(self, result, test_case, device): @@ -303,12 +347,12 @@ def __call__(self, output, test_case, device, execution_time=0): self.results[device][case_index][channel][frame][objid].append(item) else: self.results[device][case_index][channel][frame][objid].append(roi_attr_license[index]) - + #print("Device: {}- case: {} - channel: {} - frame: {}: {}".format(device, case_index, channel, frame, self.results[device][case_index][channel][frame])) self.case_index[device] += 1 DEMOS = [ deepcopy(BASE['security_barrier_camera_demo/cpp']) - .update_option({'-r': None,'-ni': '16', '-n_iqs': '1', '-i': '10_images.mp4'}) + .update_option({'-r': None,'-ni': '16', '-n_iqs': '1', '-i': '204_images.mp4'}) .add_parser(DemoSecurityBarrierCamera) ] From 65fe33b7bada4d1ccd85a2c311fc4148fc7cc759 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Fri, 6 May 2022 15:13:57 +0800 Subject: [PATCH 35/46] 1. Throw the exception when parsing raw data failed. 2. Correct the value assign logic for the output port name of vehicle attribution. Signed-off-by: Wang, Yang --- .../cpp/net_wrappers.hpp | 20 ++++--- demos/tests/correctness_cases.py | 60 ++++++++++--------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp index ccb9a181271..d136524446a 100644 --- a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp +++ b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp @@ -194,10 +194,15 @@ class VehicleAttributesClassifier { throw std::logic_error("Vehicle Attribs Network expects networks having two outputs"); } - // type is the first output - m_outputNameForType = outputs[0].get_any_name(); - // color is the second output. - m_outputNameForColor = outputs[1].get_any_name(); + // Get the names within the output layer + if(outputs[0].get_any_name().find("color") != std::string::npos) { + m_outputNameForColor = outputs[0].get_any_name(); + m_outputNameForType = outputs[1].get_any_name(); + } + else { + m_outputNameForColor = outputs[1].get_any_name(); + m_outputNameForType = outputs[0].get_any_name(); + } ov::preprocess::PrePostProcessor ppp(model); @@ -239,14 +244,14 @@ class VehicleAttributesClassifier { ov::Tensor inputTensor = inferRequest.get_tensor(m_attributesInputName); ov::Shape shape = inputTensor.get_shape(); if (m_autoResize) { - ov::Tensor frameTensor = wrapMat2Tensor(img); + ov::Tensor frameTensor = wrapMat2Tensor(img.clone()); ov::Coordinate p00({ 0, (size_t)vehicleRect.y, (size_t)vehicleRect.x, 0 }); ov::Coordinate p01({ 1, (size_t)(vehicleRect.y + vehicleRect.height), (size_t)vehicleRect.x + vehicleRect.width, 3 }); ov::Tensor roiTensor(frameTensor, p00, p01); inferRequest.set_tensor(m_attributesInputName, roiTensor); } else { - const cv::Mat& vehicleImage = img(vehicleRect); + const cv::Mat vehicleImage = img(vehicleRect).clone(); resize2tensor(vehicleImage, inputTensor); } } @@ -262,10 +267,11 @@ class VehicleAttributesClassifier { // 7 possible colors for each vehicle and we should select the one with the maximum probability ov::Tensor colorsTensor = inferRequest.get_tensor(m_outputNameForColor); const float* colorsValues = colorsTensor.data(); - + assert(7 == colorsTensor.get_size()); // 4 possible types for each vehicle and we should select the one with the maximum probability ov::Tensor typesTensor = inferRequest.get_tensor(m_outputNameForType); const float* typesValues = typesTensor.data(); + assert(4 == typesTensor.get_size()); const auto color_id = std::max_element(colorsValues, colorsValues + 7) - colorsValues; const auto type_id = std::max_element(typesValues, typesValues + 4) - typesValues; diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 8e047165380..086b2420270 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -319,35 +319,41 @@ def __call__(self, output, test_case, device, execution_time=0): self.results[device][case_index] = {} # Parsing the raw data - output = [i.rstrip() for i in output.split('\n') if "DEBUG" in i and "ChannelId" in i] - for item in output: - item = item[item.find('ChannelId'):].split(',') - # Channel ID - frame_results = {} - channel = item[0].split(':')[1] - if channel not in self.results[device][case_index]: - self.results[device][case_index][channel] = frame_results + try: + output = [i.rstrip() for i in output.split('\n') if "DEBUG" in i and "ChannelId" in i] + for item in output: + line = item + item = item[item.find('ChannelId'):].split(',') + # Channel ID + frame_results = {} + channel = item[0].split(':')[1] + if channel not in self.results[device][case_index]: + self.results[device][case_index][channel] = frame_results - # Frame ID - object_results = {} - frame = item[1].split(':')[1] - if frame not in self.results[device][case_index][channel]: - self.results[device][case_index][channel][frame] = object_results + # Frame ID + object_results = {} + frame = item[1].split(':')[1] + if frame not in self.results[device][case_index][channel]: + self.results[device][case_index][channel][frame] = object_results + + # Object ID + label_prob_pos_results = [] + objid = item[2] + if objid not in self.results[device][case_index][channel][frame]: + self.results[device][case_index][channel][frame][objid] = label_prob_pos_results + #self.results[device][case_index][channel][frame][objid] = item[3:] + roi_attr_license = ",".join(item[3:]).split('\t') + for index in range(len(roi_attr_license)): + if index == 0: + item = roi_attr_license[index].split(',') + self.results[device][case_index][channel][frame][objid].append(item) + else: + self.results[device][case_index][channel][frame][objid].append(roi_attr_license[index]) + #print("Device: {}- case: {} - channel: {} - frame: {}: {}".format(device, case_index, channel, frame, self.results[device][case_index][channel][frame])) + except IndexError: + raise IndexError ("Rawdata format is invalid:\n\t{}".format(line)) + - # Object ID - label_prob_pos_results = [] - objid = item[2] - if objid not in self.results[device][case_index][channel][frame]: - self.results[device][case_index][channel][frame][objid] = label_prob_pos_results - #self.results[device][case_index][channel][frame][objid] = item[3:] - roi_attr_license = ",".join(item[3:]).split('\t') - for index in range(len(roi_attr_license)): - if index == 0: - item = roi_attr_license[index].split(',') - self.results[device][case_index][channel][frame][objid].append(item) - else: - self.results[device][case_index][channel][frame][objid].append(roi_attr_license[index]) - #print("Device: {}- case: {} - channel: {} - frame: {}: {}".format(device, case_index, channel, frame, self.results[device][case_index][channel][frame])) self.case_index[device] += 1 DEMOS = [ From 1f44622405a59234dcf205e01b79a55563557e2f Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Sat, 7 May 2022 13:07:30 +0800 Subject: [PATCH 36/46] Add logic to check if the size of vehicle attributs is correct. Signed-off-by: Wang, Yang --- demos/security_barrier_camera_demo/cpp/net_wrappers.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp index d136524446a..c197c691df8 100644 --- a/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp +++ b/demos/security_barrier_camera_demo/cpp/net_wrappers.hpp @@ -268,10 +268,16 @@ class VehicleAttributesClassifier { ov::Tensor colorsTensor = inferRequest.get_tensor(m_outputNameForColor); const float* colorsValues = colorsTensor.data(); assert(7 == colorsTensor.get_size()); + if (colorsTensor.get_size() != 7) { + throw std::logic_error("Vehicle Color output size should be 7."); + } // 4 possible types for each vehicle and we should select the one with the maximum probability ov::Tensor typesTensor = inferRequest.get_tensor(m_outputNameForType); const float* typesValues = typesTensor.data(); assert(4 == typesTensor.get_size()); + if (typesTensor.get_size() != 4) { + throw std::logic_error("Vehicle Types output size should be 4."); + } const auto color_id = std::max_element(colorsValues, colorsValues + 7) - colorsValues; const auto type_id = std::max_element(typesValues, typesValues + 4) - typesValues; From 0ff6d8c05be6a2e35ac65bd973c0c5a33da55e41 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Mon, 9 May 2022 14:59:26 +0800 Subject: [PATCH 37/46] Update correctness checking. Signed-off-by: Wang, Yang --- demos/tests/correctness_cases.py | 6 ++++-- demos/tests/run_tests.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 086b2420270..157714d1375 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -115,7 +115,7 @@ def check_difference(self): for target in devices_list[device]: if device not in self.results or target not in self.results: multi_correctness[device][target] = {} - multi_correctness_errmsg[device]['-1'] = "\tMiss the results of device {} or device {}.\n".format(device, target) + multi_correctness_errmsg[device]['-1'] = "\tMiss the results from device {} or from device {}.\n".format(device, target) continue #if self.results[device] != self.results[target]: # # Show the detailed inconsistent results @@ -218,11 +218,12 @@ def check_difference(self): if 'MULTI:GPU,CPU' != device and 'AUTO:GPU,CPU' != device: for target in multi_correctness[device]: consistent_flag = True - err_msg = '' + err_msg = 'Inconsistent result:\n' if len(multi_correctness[device][target]) == 0: final_correctness_flag = False consistent_flag = False # Miss results for device + err_msg += '\t' err_msg += multi_correctness_errmsg[device]['-1'] else: for case in multi_correctness[device][target]: @@ -230,6 +231,7 @@ def check_difference(self): if flag == False: final_correctness_flag = False consistent_flag = False + err_msg += '\t' err_msg += multi_correctness_errmsg[device][case] if not consistent_flag: print("Checking device: {} - target : {} - : Fail.\n{}".format(device, devices_list[device], err_msg)) diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index 6e8070174b3..7d33f55ae8c 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -355,10 +355,11 @@ def option_to_args(key, value): print("Demo {} correctness checking....".format(demo.subdirectory)) if not demo.check_difference(): correctness_failures += 1 + num_failures += 1 print() if args.scope == "correctness": - print("Correctness Failures: {}".format(correctness_failures)) - print("{} failures:".format(num_failures)) + print("{} correctness checking failures".format(correctness_failures)) + print("{} execution failures:".format(num_failures)) for test in failed_tests: print(test) From 3958ea41109bec16d5003fe8f93dc0ddbf898001 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Mon, 23 May 2022 17:49:19 +0800 Subject: [PATCH 38/46] Fix the hang issue when inputting images folder. Signed-off-by: Wang, Yang --- .../security_barrier_camera_demo/cpp/main.cpp | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index 3125ecb9198..07dfa9303c6 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -407,6 +407,13 @@ void Drawer::process() { } } else { if (!context.isVideo) { + // Calculate the inference count for the inputting images. + uint32_t totalInferFrameCounter = FLAGS_ni == 0 ? FLAGS_n_iqs * context.totalFrameCount : FLAGS_ni * FLAGS_n_iqs; + if (context.totalInferFrameCounter < totalInferFrameCounter) + { + context.drawersContext.drawerMutex.unlock(); + return; + } try { std::shared_ptr(context.drawersContext.drawersWorker)->stop(); } @@ -417,6 +424,15 @@ void Drawer::process() { gridMats.emplace((--gridMats.end())->first + 1, firstGridIt->second); gridMats.erase(firstGridIt); } + if (!context.isVideo) { + uint32_t totalInferFrameCounter = FLAGS_ni == 0 ? FLAGS_n_iqs * context.totalFrameCount : FLAGS_ni * FLAGS_n_iqs; + if (context.totalInferFrameCounter == totalInferFrameCounter) { + try { + std::shared_ptr(context.drawersContext.drawersWorker)->stop(); + } + catch (const std::bad_weak_ptr&) {} + } + } context.drawersContext.drawerMutex.unlock(); } @@ -667,7 +683,14 @@ void Reader::process() { context.readersContext.lastCapturedFrameIds[sourceID]++; context.readersContext.lastCapturedFrameIdsMutexes[sourceID].unlock(); try { - if (context.totalInferFrameCounter < FLAGS_ni * context.totalFrameCount) + // Calculate the inference count for the inputting video. + uint32_t totalInferFrameCounter = 0; + if (FLAGS_ni == 0) + totalInferFrameCounter = context.totalFrameCount; + else + totalInferFrameCounter = FLAGS_ni * context.totalFrameCount; + + if (context.totalInferFrameCounter < totalInferFrameCounter) { // Rebron this invalid frame to end the worker at next time std::shared_ptr(context.drawersContext.drawersWorker)->push(std::make_shared(sharedVideoFrame)); From 42171be1b294d6328257c28dae01cb1c83d2ec91 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 25 May 2022 09:14:45 +0800 Subject: [PATCH 39/46] Update correctness checking logic to handle the exception. Signed-off-by: Wang, Yang --- demos/tests/correctness_cases.py | 129 +++++++++++++++++-------------- 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index 157714d1375..ece3dcdd0d5 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -137,69 +137,78 @@ def check_difference(self): multi_correctness[device][target][case] = False multi_correctness_errmsg[device][case] += "[Device: {}- Case: {}][Not Found on {}]Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[target][case][channel][frame]) else: - #print("======Case:{} - Device: {} - Target: {} - channel: {} - frame: {}=======".format(case, device, target, channel, frame)) - #print("Device {}: {}".format(device, self.results[device][case][channel][frame][obj])) - #print("{}: {}".format(target, self.results[target][case][channel][frame][obj])) - #if 'CPU' in device and 'GPU' in device: - # if target == 'CPU': - # print("GPU: {}".format(self.results['GPU'][case][channel][frame][obj])) - # elif target == 'GPU': - # print("CPU: {}".format(self.results['CPU'][case][channel][frame][obj])) + try: + if 'CPU' in device and 'GPU' in device: + if case not in multi_correctness_devices[device][target]: + multi_correctness_devices[device][target][case] = {} + if channel not in multi_correctness_devices[device][target][case]: + multi_correctness_devices[device][target][case][channel] = {} + if frame not in multi_correctness_devices[device][target][case][channel]: + multi_correctness_devices[device][target][case][channel][frame] = [] - if 'CPU' in device and 'GPU' in device: - if case not in multi_correctness_devices[device][target]: - multi_correctness_devices[device][target][case] = {} - if channel not in multi_correctness_devices[device][target][case]: - multi_correctness_devices[device][target][case][channel] = {} - if frame not in multi_correctness_devices[device][target][case][channel]: - multi_correctness_devices[device][target][case][channel][frame] = [] - - for i in range(len(self.results[device][case][channel][frame][obj])): - if i == 0: - # Compared ROI - device_vehicle_roi = self.results[device][case][channel][frame][obj][i] - target_vehicle_roi = self.results[target][case][channel][frame][obj][i] - flag_roi = self.compare_roi(device_vehicle_roi, target_vehicle_roi) - if 'CPU' in device and 'GPU' in device: - multi_correctness_devices[device][target][case][channel][frame].append(flag_roi) + for i in range(len(self.results[device][case][channel][frame][obj])): + if i == 0: + # Compared ROI + device_vehicle_roi = self.results[device][case][channel][frame][obj][i] + target_vehicle_roi = self.results[target][case][channel][frame][obj][i] + flag_roi = self.compare_roi(device_vehicle_roi, target_vehicle_roi) + if 'CPU' in device and 'GPU' in device: + multi_correctness_devices[device][target][case][channel][frame].append(flag_roi) + else: + if not flag_roi: + multi_correctness[device][target][case] = False + tmp_msg = ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[device][case][channel][frame])) + tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, target, channel, frame, self.results[target][case][channel][frame])) + multi_correctness_errmsg[device][case] += tmp_msg else: - if not flag_roi: - multi_correctness[device][target][case] = False - tmp_msg = ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[device][case][channel][frame])) - tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, target, channel, frame, self.results[target][case][channel][frame])) + # Compare attribute/license plate + device_vehicle_attr = self.results[device][case][channel][frame][obj][i] + target_vehicle_attr = [] + if i < len(self.results[target][case][channel][frame][obj]): + target_vehicle_attr = self.results[target][case][channel][frame][obj][i] + if 'CPU' in device and 'GPU' in device: + multi_correctness_devices[device][target][case][channel][frame].append(device_vehicle_attr == target_vehicle_attr) + else: + if device_vehicle_attr != target_vehicle_attr: + multi_correctness[device][target][case] = False + tmp_msg = ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[device][case][channel][frame])) + tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, target, channel, frame, self.results[target][case][channel][frame])) + multi_correctness_errmsg[device][case] += tmp_msg + #if 'CPU' in device and 'GPU' in device: + # print(multi_correctness_devices[device][target][case][channel][frame]) + #else: + # print(multi_correctness[device][target][case]) + #print("======================================") + if 'CPU' in device and 'GPU' in device: + # Check if correctness result between device and target for multi devices + consistent_flag = False + for flag in multi_correctness_devices[device][target][case][channel][frame]: + if flag == True: + consistent_flag = True + break + if not consistent_flag: + tmp_msg = ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, target, channel, frame, self.results[target][case][channel][frame])) + if 'CPU' in device and 'GPU' in device and target == 'CPU': + tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, 'GPU', channel, frame, self.results['GPU'][case][channel][frame])) + elif 'CPU' in device and 'GPU' in device and target == 'GPU': + tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, 'CPU', channel, frame, self.results['CPU'][case][channel][frame])) + tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[device][case][channel][frame])) multi_correctness_errmsg[device][case] += tmp_msg + except: + print("======Checking exception on Case:{} - Device: {} - Target: {} - channel: {} - frame: {}=======".format(case, device, target, channel, frame)) + print("Device {}: {}".format(device, self.results[device][case][channel][frame][obj])) + print("{}: {}".format(target, self.results[target][case][channel][frame][obj])) + if 'CPU' in device and 'GPU' in device: + multi_correctness_devices[device][target][case][channel][frame].append(False) + if target == 'CPU': + print("GPU: {}".format(self.results['GPU'][case][channel][frame][obj])) + elif target == 'GPU': + print("CPU: {}".format(self.results['CPU'][case][channel][frame][obj])) else: - # Compare attribute/license plate - device_vehicle_attr = self.results[device][case][channel][frame][obj][i] - target_vehicle_attr = self.results[target][case][channel][frame][obj][i] - if 'CPU' in device and 'GPU' in device: - multi_correctness_devices[device][target][case][channel][frame].append(device_vehicle_attr == target_vehicle_attr) - else: - if device_vehicle_attr != target_vehicle_attr: - multi_correctness[device][target][case] = False - tmp_msg = ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[device][case][channel][frame])) - tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, target, channel, frame, self.results[target][case][channel][frame])) - multi_correctness_errmsg[device][case] += tmp_msg - #if 'CPU' in device and 'GPU' in device: - # print(multi_correctness_devices[device][target][case][channel][frame]) - #else: - # print(multi_correctness[device][target][case]) - #print("======================================") - if 'CPU' in device and 'GPU' in device: - # Check if correctness result between device and target for multi devices - consistent_flag = False - for flag in multi_correctness_devices[device][target][case][channel][frame]: - if flag == True: - consistent_flag = True - break - if not consistent_flag: - tmp_msg = ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, target, channel, frame, self.results[target][case][channel][frame])) - if 'CPU' in device and 'GPU' in device and target == 'CPU': - tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, 'GPU', channel, frame, self.results['GPU'][case][channel][frame])) - elif 'CPU' in device and 'GPU' in device and target == 'GPU': - tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, 'CPU', channel, frame, self.results['CPU'][case][channel][frame])) - tmp_msg += ("[Device: {}- Case: {} on {}] Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[device][case][channel][frame])) - multi_correctness_errmsg[device][case] += tmp_msg + multi_correctness[device][target][case] = False + multi_correctness_errmsg[device][case] += "[Device: {}- Case: {}][Exception on {}]Channel {} - Frame {} : {}\n".format(device, case, device, channel, frame, self.results[target][case][channel][frame]) + + #print("=====================================") #print(multi_correctness) @@ -360,7 +369,7 @@ def __call__(self, output, test_case, device, execution_time=0): DEMOS = [ deepcopy(BASE['security_barrier_camera_demo/cpp']) - .update_option({'-r': None,'-ni': '16', '-n_iqs': '1', '-i': '204_images.mp4'}) + .update_option({'-r': None,'-ni': '16', '-n_iqs': '1', '-i': 'multi_images.mp4'}) .add_parser(DemoSecurityBarrierCamera) ] From ba629d038fa30679073bd99fa9b3c5b649b96999 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Wed, 25 May 2022 16:21:24 +0800 Subject: [PATCH 40/46] Update. Signed-off-by: Wang, Yang --- demos/tests/correctness_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/tests/correctness_cases.py b/demos/tests/correctness_cases.py index ece3dcdd0d5..627eba52761 100644 --- a/demos/tests/correctness_cases.py +++ b/demos/tests/correctness_cases.py @@ -213,7 +213,7 @@ def check_difference(self): #print("=====================================") #print(multi_correctness) #print("=====================================") - print(multi_correctness_errmsg) + #print(multi_correctness_errmsg) #print("=====================================") #for device in multi_correctness_devices: # for target in multi_correctness_devices[device]: From 6c4393338249180f17c3c577a4145be15174fd24 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Thu, 26 May 2022 14:35:10 +0800 Subject: [PATCH 41/46] Fix hange issue when inputting images folder. Signed-off-by: Wang, Yang --- .../security_barrier_camera_demo/cpp/main.cpp | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/demos/security_barrier_camera_demo/cpp/main.cpp b/demos/security_barrier_camera_demo/cpp/main.cpp index 07dfa9303c6..3bfa5f9b73a 100644 --- a/demos/security_barrier_camera_demo/cpp/main.cpp +++ b/demos/security_barrier_camera_demo/cpp/main.cpp @@ -334,6 +334,15 @@ bool Drawer::isReady() { if (std::chrono::steady_clock::now() - prevShow > showPeriod) { return true; } else { + if (!context.isVideo) { + uint32_t totalInferFrameCounter = FLAGS_ni == 0 ? FLAGS_n_iqs * context.totalFrameCount : FLAGS_ni * FLAGS_n_iqs; + if (context.totalInferFrameCounter == totalInferFrameCounter) { + try { + std::shared_ptr(context.drawersContext.drawersWorker)->stop(); + } + catch (const std::bad_weak_ptr&) {} + } + } return false; } } else { @@ -343,6 +352,15 @@ bool Drawer::isReady() { if (2 > gridMats.size()) { // buffer size return true; } else { + if (!context.isVideo) { + uint32_t totalInferFrameCounter = FLAGS_ni == 0 ? FLAGS_n_iqs * context.totalFrameCount : FLAGS_ni * FLAGS_n_iqs; + if (context.totalInferFrameCounter == totalInferFrameCounter) { + try { + std::shared_ptr(context.drawersContext.drawersWorker)->stop(); + } + catch (const std::bad_weak_ptr&) {} + } + } return false; } } else { @@ -351,6 +369,15 @@ bool Drawer::isReady() { && std::chrono::steady_clock::now() - prevShow > showPeriod) { return true; } else { + if (!context.isVideo) { + uint32_t totalInferFrameCounter = FLAGS_ni == 0 ? FLAGS_n_iqs * context.totalFrameCount : FLAGS_ni * FLAGS_n_iqs; + if (context.totalInferFrameCounter == totalInferFrameCounter) { + try { + std::shared_ptr(context.drawersContext.drawersWorker)->stop(); + } + catch (const std::bad_weak_ptr&) {} + } + } return false; } } else { From 5ccace47a2085e82400e27987a4effd0e2919da0 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Tue, 28 Jun 2022 10:25:18 +0800 Subject: [PATCH 42/46] Fix the run_tests.py terminated with exception when timeout occurs. Signed-off-by: Wang, Yang --- demos/tests/run_tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index e05391cd379..2636dd517e0 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -333,7 +333,10 @@ def option_to_args(key, value): execution_time = timeit.default_timer() - start_time demo.parse_output(output, test_case, device) except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - output = e.output + if type(e) == subprocess.TimeoutExpired: + output = e.output.decode('utf-8') + else: + output = e.output if isinstance(e, subprocess.CalledProcessError): exit_msg = f'Exit code: {e.returncode}\n' elif isinstance(e, subprocess.TimeoutExpired): From a5d84dca41f7b54e28e1eddff9d868d6683533b7 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Mon, 15 Aug 2022 10:56:17 +0800 Subject: [PATCH 43/46] Update. Signed-off-by: Wang, Yang --- demos/tests/run_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index 7d33f55ae8c..1dade7535c2 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -326,6 +326,7 @@ def option_to_args(key, value): for arg in fixed_args + dev_arg + case_args)) print(test_descr) print(flush=True) + execution_time = 1.0 try: start_time = timeit.default_timer() output = subprocess.check_output(fixed_args + dev_arg + case_args, From 7c57fd229ef0f307589f04d7d62b136d9cbd51d5 Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Tue, 16 Aug 2022 09:23:42 +0800 Subject: [PATCH 44/46] update. Signed-off-by: Wang, Yang --- demos/tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index 1dade7535c2..9856d638290 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -326,7 +326,6 @@ def option_to_args(key, value): for arg in fixed_args + dev_arg + case_args)) print(test_descr) print(flush=True) - execution_time = 1.0 try: start_time = timeit.default_timer() output = subprocess.check_output(fixed_args + dev_arg + case_args, @@ -344,6 +343,7 @@ def option_to_args(key, value): print(output) failed_tests.append(test_descr + '\n' + exit_msg) num_failures += 1 + execution_time = -1 if args.report_file: collect_result(demo.subdirectory, device, case_model_names, execution_time, args.report_file) From f7d7a1dd5d347f69d41a0427525fbe7f6a9fcabc Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Tue, 13 Sep 2022 13:04:14 +0800 Subject: [PATCH 45/46] Update. Signed-off-by: Wang, Yang --- demos/tests/run_tests.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index 2636dd517e0..c82da1f5066 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -333,10 +333,9 @@ def option_to_args(key, value): execution_time = timeit.default_timer() - start_time demo.parse_output(output, test_case, device) except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - if type(e) == subprocess.TimeoutExpired: + output = e.output + if not isinstance(output, str): output = e.output.decode('utf-8') - else: - output = e.output if isinstance(e, subprocess.CalledProcessError): exit_msg = f'Exit code: {e.returncode}\n' elif isinstance(e, subprocess.TimeoutExpired): From 19f9ff1052723573f676fa6dfbf6eea2ec9d4ceb Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Tue, 13 Sep 2022 13:30:37 +0800 Subject: [PATCH 46/46] Update. Signed-off-by: Wang, Yang --- demos/tests/run_tests.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/demos/tests/run_tests.py b/demos/tests/run_tests.py index c82da1f5066..079af98de3c 100755 --- a/demos/tests/run_tests.py +++ b/demos/tests/run_tests.py @@ -334,14 +334,11 @@ def option_to_args(key, value): demo.parse_output(output, test_case, device) except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: output = e.output - if not isinstance(output, str): - output = e.output.decode('utf-8') if isinstance(e, subprocess.CalledProcessError): exit_msg = f'Exit code: {e.returncode}\n' elif isinstance(e, subprocess.TimeoutExpired): exit_msg = f'Command timed out after {e.timeout} seconds\n' - output += exit_msg - print(output) + print('{}\n{}'.format(output, exit_msg)) failed_tests.append(test_descr + '\n' + exit_msg) num_failures += 1 execution_time = -1