Move PCF2-Lift Options to a shared location (#2198)

Summary:
Pull Request resolved: #2198

This is moving the Lift options out of main so that they can be used in other parts of the code. This is especially important for feature flagging within the cpp code.

Reviewed By: ajinkya-ghonge, haochenuw

Differential Revision: D44062784

fbshipit-source-id: 46b4dc7fc95f14d24eeed32462569f4c3ffad9fe

Author: Caleb Jaeger

fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp

@@ -24,8 +24,6 @@
 constexpr int32_t tsOffset = 10;
 
 DEFINE_bool(is_conversion_lift, true, "is conversion lift");
-DEFINE_int32(num_conversions_per_user, 4, "num of conversions per user");
-DEFINE_int64(epoch, 1546300800, "epoch");
 
 namespace private_lift {
 class CalculatorAppTest : public ::testing::Test {

fbpcs/emp_games/lift/common/CommonLiftOptions.cpp

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h"
+#include <gflags/gflags.h>
+
+DEFINE_bool(
+    compute_publisher_breakdowns,
+    true,
+    "To enable or disable computing publisher breakdown for result validation");
+DEFINE_bool(
+    log_cost,
+    false,
+    "Log cost info into cloud which will be used for dashboard");
+DEFINE_bool(
+    use_tls,
+    false,
+    "Whether to use TLS when communicating with other parties.");
+DEFINE_bool(
+    use_xor_encryption,
+    true,
+    "Reveal output with XOR secret shares instead of in the clear to both parties");
+DEFINE_int32(concurrency, 1, "max number of games that will run concurrently");
+DEFINE_int32(
+    file_start_index,
+    0,
+    "First file that will be read with base path");
+DEFINE_int32(
+    num_conversions_per_user,
+    4,
+    "Cap and pad to this many conversions per user");
+DEFINE_int32(num_files, 0, "Number of files that should be read");
+DEFINE_int32(party, 1, "1 = publisher, 2 = partner");
+DEFINE_int32(
+    port,
+    10000,
+    "Network port for establishing connection to other player");
+DEFINE_int64(
+    epoch,
+    1546300800,
+    "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps");
+DEFINE_string(
+    ca_cert_path,
+    "",
+    "Relative file path where root CA cert is stored. It will be prefixed with $HOME.");
+DEFINE_string(
+    input_base_path,
+    "",
+    "Local or s3 base path for the sharded input files");
+DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name");
+DEFINE_string(
+    log_cost_s3_region,
+    ".s3.us-west-2.amazonaws.com/",
+    "s3 region name");
+DEFINE_string(
+    pc_feature_flags,
+    "",
+    "A String of PC Feature Flags passing from PCS, separated by comma");
+DEFINE_string(
+    private_key_path,
+    "",
+    "Relative file path where private key is stored. It will be prefixed with $HOME.");
+DEFINE_string(
+    run_name,
+    "",
+    "A user given run name that will be used in s3 filename");
+DEFINE_string(
+    server_cert_path,
+    "",
+    "Relative file path where server cert is stored. It will be prefixed with $HOME.");
+DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address");

fbpcs/emp_games/lift/common/CommonLiftOptions.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <gflags/gflags_declare.h>
+
+DECLARE_bool(compute_publisher_breakdowns);
+DECLARE_bool(log_cost);
+DECLARE_bool(use_tls);
+DECLARE_bool(use_xor_encryption);
+DECLARE_int32(concurrency);
+DECLARE_int32(file_start_index);
+DECLARE_int32(num_conversions_per_user);
+DECLARE_int32(num_files);
+DECLARE_int32(party);
+DECLARE_int32(port);
+DECLARE_int64(epoch);
+DECLARE_string(ca_cert_path);
+DECLARE_string(input_base_path);
+DECLARE_string(log_cost_s3_bucket);
+DECLARE_string(log_cost_s3_region);
+DECLARE_string(pc_feature_flags);
+DECLARE_string(private_key_path);
+DECLARE_string(run_name);
+DECLARE_string(server_cert_path);
+DECLARE_string(server_ip);

fbpcs/emp_games/lift/metadata_compaction/MainUtil.h

@@ -8,13 +8,13 @@
 #pragma once
 
 #include <memory>
+#include <vector>
 #include "folly/Format.h"
 
 #include "fbpcf/engine/communication/SocketPartyCommunicationAgentFactory.h"
 #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorApp.h"
 #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorGame.h"
 #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorGameFactory.h"
-
 namespace private_lift {
 
 struct LiftMetadataCompactionFilePaths {

fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp

@@ -6,93 +6,89 @@
  */
 
 #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h"
+#include <gflags/gflags.h>
+#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h"
 
-DEFINE_int32(party, 1, "1 = publisher, 2 = partner");
-DEFINE_bool(
-    use_xor_encryption,
-    true,
-    "Reveal output with XOR secret shares instead of in the clear to both parties");
-DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address");
-DEFINE_int32(
-    port,
-    10000,
-    "Network port for establishing connection to other player");
-
-// Lift settings
 DEFINE_string(input_path, "", "Input file to run lift metadata compaction");
+DEFINE_string(
+    output_global_params_base_path,
+    "",
+    "Local or s3 base path where output global param files are written to");
 DEFINE_string(
     output_global_params_path,
     "",
     "Output file to write global params from input data.");
+DEFINE_string(
+    output_secret_shares_base_path,
+    "",
+    "Local or s3 base path where output secret share files are written to");
 DEFINE_string(
     output_secret_shares_path,
     "",
     "Output file to write compacted metadata secret share results.");
+// Common lift options
+DEFINE_bool(
+    compute_publisher_breakdowns,
+    true,
+    "To enable or disable computing publisher breakdown for result validation");
+DEFINE_bool(
+    log_cost,
+    false,
+    "Log cost info into cloud which will be used for dashboard");
+DEFINE_bool(
+    use_tls,
+    false,
+    "Whether to use TLS when communicating with other parties.");
+DEFINE_bool(
+    use_xor_encryption,
+    true,
+    "Reveal output with XOR secret shares instead of in the clear to both parties");
+DEFINE_int32(concurrency, 1, "max number of games that will run concurrently");
 DEFINE_int32(
     file_start_index,
     0,
     "First file that will be read with base path");
+DEFINE_int32(
+    num_conversions_per_user,
+    4,
+    "Cap and pad to this many conversions per user");
 DEFINE_int32(num_files, 0, "Number of files that should be read");
-DEFINE_string(
-    input_base_path,
-    "",
-    "Local or s3 base path for the sharded input files");
-DEFINE_string(
-    output_global_params_base_path,
-    "",
-    "Local or s3 base path where output global param files are written to");
-DEFINE_string(
-    output_secret_shares_base_path,
-    "",
-    "Local or s3 base path where output secret share files are written to");
-DEFINE_int32(concurrency, 1, "max number of games that will run concurrently");
+DEFINE_int32(party, 1, "1 = publisher, 2 = partner");
 DEFINE_int32(
+    port,
+    10000,
+    "Network port for establishing connection to other player");
+DEFINE_int64(
     epoch,
     1546300800,
     "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps");
-DEFINE_int32(
-    num_conversions_per_user,
-    4,
-    "Cap and pad to this many conversions per user");
-DEFINE_bool(
-    compute_publisher_breakdowns,
-    true,
-    "To enable or disable computing publisher breakdown for result validation");
-
-// TLS Settings
-DEFINE_bool(
-    use_tls,
-    false,
-    "Whether to use TLS when communicating with other parties.");
 DEFINE_string(
     ca_cert_path,
     "",
     "Relative file path where root CA cert is stored. It will be prefixed with $HOME.");
 DEFINE_string(
-    server_cert_path,
+    input_base_path,
     "",
-    "Relative file path where server cert is stored. It will be prefixed with $HOME.");
+    "Local or s3 base path for the sharded input files");
+DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name");
+DEFINE_string(
+    log_cost_s3_region,
+    ".s3.us-west-2.amazonaws.com/",
+    "s3 region name");
+DEFINE_string(
+    pc_feature_flags,
+    "",
+    "A String of PC Feature Flags passing from PCS, separated by comma");
 DEFINE_string(
     private_key_path,
     "",
     "Relative file path where private key is stored. It will be prefixed with $HOME.");
-
-// Logging flags
 DEFINE_string(
     run_name,
     "",
     "A user given run name that will be used in s3 filename");
-DEFINE_bool(
-    log_cost,
-    false,
-    "Log cost info into cloud which will be used for dashboard");
-DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name");
-DEFINE_string(
-    log_cost_s3_region,
-    ".s3.us-west-2.amazonaws.com/",
-    "s3 region name");
-
 DEFINE_string(
-    pc_feature_flags,
+    server_cert_path,
     "",
-    "A String of PC Feature Flags passing from PCS, separated by comma");
+    "Relative file path where server cert is stored. It will be prefixed with $HOME.");
+DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address");

fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h

@@ -9,36 +9,9 @@
 
 #include <gflags/gflags.h>
 
-// MPC settings
-DECLARE_int32(party);
-DECLARE_bool(use_xor_encryption);
-DECLARE_string(server_ip);
-DECLARE_int32(port);
-
 // Lift settings
 DECLARE_string(input_path);
-DECLARE_string(output_global_params_path);
-DECLARE_string(output_secret_shares_path);
-DECLARE_int32(file_start_index);
-DECLARE_int32(num_files);
-DECLARE_string(input_base_path);
 DECLARE_string(output_global_params_base_path);
+DECLARE_string(output_global_params_path);
 DECLARE_string(output_secret_shares_base_path);
-DECLARE_int32(concurrency);
-DECLARE_int32(epoch);
-DECLARE_int32(num_conversions_per_user);
-DECLARE_bool(compute_publisher_breakdowns);
-
-// TLS Settings
-DECLARE_bool(use_tls);
-DECLARE_string(ca_cert_path);
-DECLARE_string(server_cert_path);
-DECLARE_string(private_key_path);
-
-// Logging flags
-DECLARE_string(run_name);
-DECLARE_bool(log_cost);
-DECLARE_string(log_cost_s3_bucket);
-DECLARE_string(log_cost_s3_region);
-
-DECLARE_string(pc_feature_flags);
+DECLARE_string(output_secret_shares_path);

fbpcs/emp_games/lift/metadata_compaction/main.cpp

@@ -13,8 +13,10 @@
 #include "fbpcf/aws/AwsSdk.h"
 #include "fbpcs/performance_tools/CostEstimation.h"
 
+#include <gflags/gflags.h>
 #include "fbpcs/emp_games/common/Constants.h"
 #include "fbpcs/emp_games/common/Util.h"
+#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h"
 #include "fbpcs/emp_games/lift/metadata_compaction/MainUtil.h"
 #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h"
 

fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include "fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h"
+#include <gflags/gflags.h>
+
+DEFINE_bool(
+    is_conversion_lift,
+    true,
+    "Use conversion_lift logic (as opposed to converter_lift logic)");
+DEFINE_string(
+    input_directory,
+    "",
+    "Data directory where input files are located");
+DEFINE_string(
+    input_expanded_key_path,
+    "out.csv_expanded_key_0",
+    "Input file name of the expanded key for UDP decryption. Used when decoupled UDP is enabled.");
+DEFINE_string(
+    input_filenames,
+    "in.csv_0[,in.csv_1,in.csv_2,...]",
+    "List of input file names that should be parsed (should have a header)");
+DEFINE_string(
+    input_global_params_path,
+    "out.csv_global_params_0",
+    "Input file name of global parameter setup. Used when reading inputs in secret share format rather than plaintext.");
+DEFINE_string(
+    output_base_path,
+    "",
+    "Local or s3 base path where output files are written to");
+DEFINE_string(
+    output_directory,
+    "",
+    "Local or s3 path where output files are written to");
+DEFINE_string(
+    output_filenames,
+    "out.csv_0[,out.csv_1,out.csv_2,...]",
+    "List of output file names that correspond to input filenames (positionally)");
+DEFINE_string(
+    run_id,
+    "",
+    "A run_id used to identify all the logs in a PL run.");

fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <gflags/gflags_declare.h>
+#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h"
+
+DECLARE_bool(is_conversion_lift);
+DECLARE_string(input_directory);
+DECLARE_string(input_expanded_key_path);
+DECLARE_string(input_filenames);
+DECLARE_string(input_global_params_path);
+DECLARE_string(output_base_path);
+DECLARE_string(output_directory);
+DECLARE_string(output_filenames);
+DECLARE_string(run_id);