From b5a99e2174f115f87c0cb3de9d2a2479a0193db2 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Sun, 18 Nov 2018 13:11:38 +0800 Subject: [PATCH 1/3] add cmdline option to allow running fp32 models with fp16 Add an option for TFLite benchmark_model to allow running fp32 models with fp16. Useful when testing NNPAI accelerators with fp16. --- .../lite/tools/benchmark/benchmark_tflite_model.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc index 83e0ff1f872ef7..2d8b3a198c2967 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc +++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc @@ -200,6 +200,7 @@ BenchmarkParams BenchmarkTfLiteModel::DefaultParams() { default_params.AddParam("input_layer_shape", BenchmarkParam::Create("")); default_params.AddParam("use_nnapi", BenchmarkParam::Create(false)); + default_params.AddParam("allow_fp16", BenchmarkParam::Create(false)); return default_params; } @@ -219,7 +220,8 @@ std::vector BenchmarkTfLiteModel::GetFlags() { CreateFlag("input_layer", ¶ms_, "input layer names"), CreateFlag("input_layer_shape", ¶ms_, "input layer shape"), - CreateFlag("use_nnapi", ¶ms_, "use nnapi api")}; + CreateFlag("use_nnapi", ¶ms_, "use nnapi api"), + CreateFlag("allow_fp16", ¶ms_, "allow fp16")}; flags.insert(flags.end(), specific_flags.begin(), specific_flags.end()); return flags; @@ -233,6 +235,7 @@ void BenchmarkTfLiteModel::LogParams() { TFLITE_LOG(INFO) << "Input shapes: [" << params_.Get("input_layer_shape") << "]"; TFLITE_LOG(INFO) << "Use nnapi : [" << params_.Get("use_nnapi") << "]"; + TFLITE_LOG(INFO) << "Allow fp16 : [" << params_.Get("allow_fp16") << "]"; } bool BenchmarkTfLiteModel::ValidateParams() { @@ -328,6 +331,10 @@ void BenchmarkTfLiteModel::Init() { interpreter->UseNNAPI(use_nnapi); ApplyDelegates(); + bool allow_fp16 = params_.Get("allow_fp16"); + + interpreter->SetAllowFp16PrecisionForFp32(allow_fp16); + auto interpreter_inputs = interpreter->inputs(); if (!inputs.empty()) { From dc4b9745b41ca7b2a00efb142bb0ab1e9895b4e0 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Sun, 18 Nov 2018 13:31:02 +0800 Subject: [PATCH 2/3] add option for label_image to allow running fp32 models with fp16 Add an option for TFLite label_image to allow running fp32 models with fp16. Useful when testing NNPAI accelerators with fp16. --- tensorflow/lite/examples/label_image/label_image.cc | 6 ++++++ tensorflow/lite/examples/label_image/label_image.h | 1 + 2 files changed, 7 insertions(+) diff --git a/tensorflow/lite/examples/label_image/label_image.cc b/tensorflow/lite/examples/label_image/label_image.cc index b8dc2840dfb49f..b83563280ec173 100644 --- a/tensorflow/lite/examples/label_image/label_image.cc +++ b/tensorflow/lite/examples/label_image/label_image.cc @@ -113,6 +113,7 @@ void RunInference(Settings* s) { } interpreter->UseNNAPI(s->accel); + interpreter->SetAllowFp16PrecisionForFp32(s->allow_fp16); if (s->verbose) { LOG(INFO) << "tensors size: " << interpreter->tensors_size() << "\n"; @@ -253,6 +254,7 @@ void RunInference(Settings* s) { void display_usage() { LOG(INFO) << "label_image\n" << "--accelerated, -a: [0|1], use Android NNAPI or not\n" + << "--allow_fp16, -f: [0|1], allow running fp32 models with fp16 not\n" << "--count, -c: loop interpreter->Invoke() for certain times\n" << "--input_mean, -b: input mean\n" << "--input_std, -s: input standard deviation\n" @@ -273,6 +275,7 @@ int Main(int argc, char** argv) { while (1) { static struct option long_options[] = { {"accelerated", required_argument, nullptr, 'a'}, + {"allow_fp16", required_argument, nullptr, 'f'}, {"count", required_argument, nullptr, 'c'}, {"verbose", required_argument, nullptr, 'v'}, {"image", required_argument, nullptr, 'i'}, @@ -305,6 +308,9 @@ int Main(int argc, char** argv) { s.loop_count = strtol(optarg, nullptr, 10); // NOLINT(runtime/deprecated_fn) break; + case 'f': + s.allow_fp16 = strtol(optarg, nullptr, 10); // NOLINT(runtime/deprecated_fn) + break; case 'i': s.input_bmp_name = optarg; break; diff --git a/tensorflow/lite/examples/label_image/label_image.h b/tensorflow/lite/examples/label_image/label_image.h index 88b047fecc4b3e..cc46e56b64a9dc 100644 --- a/tensorflow/lite/examples/label_image/label_image.h +++ b/tensorflow/lite/examples/label_image/label_image.h @@ -26,6 +26,7 @@ struct Settings { bool accel = false; bool input_floating = false; bool profiling = false; + bool allow_fp16 = false; int loop_count = 1; float input_mean = 127.5f; float input_std = 127.5f; From ab7b554405dc4bf5b023522095a9b4a318327e0b Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 12 Feb 2019 16:15:22 +0800 Subject: [PATCH 3/3] rebase and run clang-format --- .../lite/examples/label_image/label_image.cc | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/tensorflow/lite/examples/label_image/label_image.cc b/tensorflow/lite/examples/label_image/label_image.cc index b83563280ec173..340fbab5c6fcc9 100644 --- a/tensorflow/lite/examples/label_image/label_image.cc +++ b/tensorflow/lite/examples/label_image/label_image.cc @@ -252,20 +252,21 @@ void RunInference(Settings* s) { } void display_usage() { - LOG(INFO) << "label_image\n" - << "--accelerated, -a: [0|1], use Android NNAPI or not\n" - << "--allow_fp16, -f: [0|1], allow running fp32 models with fp16 not\n" - << "--count, -c: loop interpreter->Invoke() for certain times\n" - << "--input_mean, -b: input mean\n" - << "--input_std, -s: input standard deviation\n" - << "--image, -i: image_name.bmp\n" - << "--labels, -l: labels for the model\n" - << "--tflite_model, -m: model_name.tflite\n" - << "--profiling, -p: [0|1], profiling or not\n" - << "--num_results, -r: number of results to show\n" - << "--threads, -t: number of threads\n" - << "--verbose, -v: [0|1] print more information\n" - << "\n"; + LOG(INFO) + << "label_image\n" + << "--accelerated, -a: [0|1], use Android NNAPI or not\n" + << "--allow_fp16, -f: [0|1], allow running fp32 models with fp16 not\n" + << "--count, -c: loop interpreter->Invoke() for certain times\n" + << "--input_mean, -b: input mean\n" + << "--input_std, -s: input standard deviation\n" + << "--image, -i: image_name.bmp\n" + << "--labels, -l: labels for the model\n" + << "--tflite_model, -m: model_name.tflite\n" + << "--profiling, -p: [0|1], profiling or not\n" + << "--num_results, -r: number of results to show\n" + << "--threads, -t: number of threads\n" + << "--verbose, -v: [0|1] print more information\n" + << "\n"; } int Main(int argc, char** argv) { @@ -309,7 +310,8 @@ int Main(int argc, char** argv) { strtol(optarg, nullptr, 10); // NOLINT(runtime/deprecated_fn) break; case 'f': - s.allow_fp16 = strtol(optarg, nullptr, 10); // NOLINT(runtime/deprecated_fn) + s.allow_fp16 = + strtol(optarg, nullptr, 10); // NOLINT(runtime/deprecated_fn) break; case 'i': s.input_bmp_name = optarg;