Write out optical size diagnostic (#1597)

* Rename diagnostic for consistency
* Add optical size diagnostic
* Only add optical loop parameters when running an optical problem
* Don't write optical loop input when disabled
* Fix app test names
* Add assertions for sizes
* Account for multithreading
* Make initializers more reproducible
* Improve debug message and doc
* Fix build errors with old GCC/Geant4/nljson (const char* was ambiguous)
* Use divisible-into-100 events

Author: sethrj

app/celer-export-geant.cc

@@ -15,8 +15,6 @@
 #include <vector>
 #include <nlohmann/json.hpp>
 
-#include "corecel/Config.hh"
-
 #include "corecel/Assert.hh"
 #include "corecel/io/Logger.hh"
 #include "corecel/io/StringUtils.hh"

app/celer-sim/CMakeLists.txt

@@ -76,11 +76,12 @@ function(celer_sim_test test_ext gdml_inp_name hepmc3_inp_name mctruth_name)
     set(_needs_deps DISABLED true)
   endif()
 
-  add_test(NAME "app/celer-sim-${test_ext}:gpu"
+  set(_name "app/celer-sim:${test_ext}:gpu")
+  add_test(NAME "${_name}"
     COMMAND "${CELER_PYTHON}"
     "${_driver}" "${_gdml_inp}" "${_hepmc3_inp}" "${_mctruth_out}"
   )
-  set_tests_properties("app/celer-sim-${test_ext}:gpu" PROPERTIES
+  set_tests_properties("${_name}" PROPERTIES
     ENVIRONMENT "${_env};${CELER_G4ENV}"
     RESOURCE_LOCK gpu
     REQUIRED_FILES "${_driver};${_gdml_inp};${_hepmc3_inp}"
@@ -100,7 +101,8 @@ function(celer_sim_test test_ext gdml_inp_name hepmc3_inp_name mctruth_name)
     set(CELER_GEO "${CELER_CORE_GEO}-JSON")
   endif()
 
-  add_test(NAME "app/celer-sim-${test_ext}:cpu"
+  set(_name "app/celer-sim:${test_ext}:cpu")
+  add_test(NAME "${_name}"
     COMMAND "${CELER_PYTHON}"
     "${_driver}" "${_gdml_inp}" "${_hepmc3_inp}" "${_mctruth_out}"
   )
@@ -118,7 +120,7 @@ function(celer_sim_test test_ext gdml_inp_name hepmc3_inp_name mctruth_name)
     set(CELER_OMP_ENV "OMP_NUM_THREADS=1")
     set(CELER_PROCESSORS)
   endif()
-  set_tests_properties("app/celer-sim-${test_ext}:cpu" PROPERTIES
+  set_tests_properties("${_name}" PROPERTIES
     ENVIRONMENT "${_env};${CELER_G4ENV};${CELER_OMP_ENV}"
     REQUIRED_FILES "${_driver};${_gdml_inp};${_hepmc3_inp}"
     LABELS "app;nomemcheck"
@@ -130,13 +132,13 @@ endfunction()
 #-----------------------------------------------------------------------------#
 
 celer_sim_test(cms
-  simple-cms.gdml gamma-3evt-15prim.hepmc3 simple-cms-mctruth.root
+  simple-cms.gdml gamma-4evt-15prim.hepmc3 simple-cms-mctruth.root
 )
 
 # TODO: Currently simple-driver.py uses the same Runner input parameters on
 #       both celer-sim-cms and celer-sim-lar tests. Some may need to be updated.
 celer_sim_test(lar
-  lar-sphere.gdml gamma-3evt-15prim.hepmc3 lar-mctruth.root
+  lar-sphere.gdml gamma-4evt-15prim.hepmc3 lar-mctruth.root
 )
 
 #-----------------------------------------------------------------------------#

app/celer-sim/RunnerInput.hh

@@ -101,7 +101,7 @@ struct RunnerInput
     // Control
     unsigned int seed{};
     size_type num_track_slots{};  //!< Divided among streams
-    size_type max_steps = static_cast<size_type>(-1);
+    size_type max_steps = static_cast<size_type>(-1);  //!< Step *iterations*
     size_type initializer_capacity{};  //!< Divided among streams
     real_type secondary_stack_factor{};
     bool use_device{};

app/celer-sim/RunnerInputIO.json.cc

@@ -200,7 +200,7 @@ void to_json(nlohmann::json& j, RunnerInput const& v)
                    v.physics_file.empty()
                        || !ends_with(v.physics_file, ".root"));
 
-    LDIO_SAVE(optical);
+    LDIO_SAVE_WHEN(optical, v.optical);
 
 #undef LDIO_SAVE_OPTION
 #undef LDIO_SAVE_WHEN

app/celer-sim/simple-driver.py

@@ -86,13 +86,6 @@ def strtobool(text):
     # shorten to an unreasonably small number to reduce test time.
     max_steps = 256
 
-# TODO: Update once tracking loop is implemented
-optical_options = {
-    'buffer_capacity': 3 * max_steps * num_tracks,
-    'primary_capacity': num_tracks,
-    'auto_flush': 2**31, # Large enough to never launch optical loop
-}
-
 inp = {
     'use_device': use_device,
     'geometry_file': geometry_filename,
@@ -102,7 +95,7 @@ def strtobool(text):
     'seed': 12345,
     'num_track_slots': num_tracks,
     'max_steps': max_steps,
-    'initializer_capacity': 100 * max([num_tracks, num_primaries]),
+    'initializer_capacity': 100 * num_tracks,
     'secondary_stack_factor': 3,
     'action_diagnostic': True,
     'step_diagnostic': True,
@@ -115,10 +108,19 @@ def strtobool(text):
     'brem_combined': True,
     'physics_options': physics_options,
     'field': None,
-    'optical': optical_options,
     'slot_diagnostic_prefix': f"slot-diag-{run_name}-",
 }
 
+if "lar" in geometry_filename:
+    inp['optical'] = {
+        'buffer_capacity': 3 * max_steps * num_tracks,
+        'primary_capacity': num_tracks,
+        'auto_flush': 2**31, # Large enough to never launch optical loop
+    }
+
+if physics_filename:
+    inp['physics_file'] = physics_filename
+
 inp_file = f'{run_name}.inp.json'
 with open(inp_file, 'w') as f:
     json.dump(inp, f, indent=1)
@@ -159,14 +161,36 @@ def strtobool(text):
     json.dump(j, f, indent=1)
 print("Results written to", out_file, file=stderr)
 
-run_output =j['result']['runner']
+run_output = j['result']['runner']
 time = run_output['time'].copy()
 steps = time.pop('steps')
 if use_device:
     assert steps
-    assert len(steps[0]) == run_output['num_step_iterations'][0]
+    assert len(steps[0]) == run_output['num_step_iterations'][0], steps[0]
 else:
     # Step times disabled on CPU from input
-    assert steps is None
+    assert steps is None, steps
+
+internal = j["internal"]
+if "lar" in geometry_filename and not use_device:
+    core_sizes = internal["core-sizes"].copy()
+    num_streams = internal["core-sizes"].pop("streams")
+    if "openmp" not in j["system"]["build"]["config"]["use"]:
+        assert num_streams == 1
+    assert internal["core-sizes"] == {
+       "events": 4,
+       "initializers": 3200,
+       "processes": 1,
+       "secondaries": 96,
+       "tracks": 32
+      }, core_sizes
+
+    opt_sizes = internal["optical-sizes"].copy()
+    assert num_streams == opt_sizes.pop("streams")
+    assert opt_sizes == {
+       "generators": 24576 * num_streams,
+       "initializers": 32 * num_streams,
+       "tracks": 0,
+      }, opt_sizes
 
 print(json.dumps(time, indent=1))

app/data/gamma-3evt-15prim.hepmc3 app/data/gamma-4evt-15prim.hepmc3

@@ -1,25 +1,27 @@
 HepMC::Version 3.02.02
 HepMC::Asciiv3-START_EVENT_LISTING
-E 0 0 5
+E 0 0 3
 U MEV CM
 P 1 0 22 5.1986662883182282e+02 -4.2922054653912312e+02 -7.3858541188930360e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
 P 2 0 22 7.3395459362460701e+02 1.8726575230281321e+02 6.5287226354915663e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
 P 3 0 22 -4.0053358241288629e+02 -8.1839341451526892e+01 9.1261994913012950e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
-P 4 0 22 -5.1571621404849338e+02 1.2578032388599880e+02 8.4747631040084048e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
-P 5 0 22 -5.0829382297517975e+02 5.1523183959000494e+02 -6.9005328852051423e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
-E 1 0 5
+E 1 0 4
 U MEV CM
 P 1 0 22 2.5183128938864547e+02 -2.0216120822226654e+02 -9.4642054477645695e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
 P 2 0 22 -2.5247976713163627e+02 9.4617275706343776e+02 -2.0251192799469257e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
 P 3 0 22 3.4066344768752202e+02 -9.0517210955885571e+02 2.5418864547107884e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
 P 4 0 22 8.3192692739205529e+02 -5.4330006880869769e+02 1.1279460409291821e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
-P 5 0 22 2.3445050379268079e+02 -3.6984950141988622e+02 -8.9902408620171002e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
-E 2 0 5
+E 2 0 4
+U MEV CM
+P 1 0 22 1.7562103525403742e+02 -4.7618127524473937e+02 8.6163138585046556e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
+P 2 0 22 -6.0694965222663632e+02 6.9697036165837198e+02 3.8189584264791790e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
+P 3 0 22 5.1336099422574807e+02 5.4197742781708928e+02 6.6537279576514345e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
+P 4 0 22 -3.6655746358147650e+02 8.0035990693978067e+02 4.7440451647940733e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
+E 3 0 4
 U MEV CM
 P 1 0 22 1.7562103525403742e+02 -4.7618127524473937e+02 8.6163138585046556e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
 P 2 0 22 -6.0694965222663632e+02 6.9697036165837198e+02 3.8189584264791790e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
 P 3 0 22 5.1336099422574807e+02 5.4197742781708928e+02 6.6537279576514345e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
 P 4 0 22 -3.6655746358147650e+02 8.0035990693978067e+02 4.7440451647940733e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
-P 5 0 22 -7.8969793730748802e+02 -5.4961247282688134e+02 -2.7258631206540650e+02 1.0000000000000000e+03 0.0000000000000000e+00 1
 HepMC::Asciiv3-END_EVENT_LISTING
 

src/accel/detail/HitManagerOutput.cc

@@ -59,7 +59,7 @@ void HitManagerOutput::output(JsonPimpl* j) const
             G4VSensitiveDetector const* sd{nullptr};
             if (lv)
             {
-                gv_names.push_back(lv->GetName());
+                gv_names.push_back(std::string(lv->GetName()));
                 sd = lv->GetSensitiveDetector();
             }
             else
@@ -69,7 +69,7 @@ void HitManagerOutput::output(JsonPimpl* j) const
 
             if (sd)
             {
-                sd_names.push_back(sd->GetName());
+                sd_names.push_back(std::string(sd->GetName()));
                 sd_types.push_back(demangle_sd(*sd));
             }
             else

src/celeritas/Types.cc

@@ -9,8 +9,6 @@
 
 #include "corecel/io/EnumStringMapper.hh"
 
-#include "UnitTypes.hh"
-
 namespace celeritas
 {
 //---------------------------------------------------------------------------//

src/celeritas/global/CoreParams.cc

@@ -341,7 +341,7 @@ CoreParams::CoreParams(Input input) : input_(std::move(input))
     input_.output_reg->insert(
         OutputInterfaceAdapter<detail::CoreSizes>::from_rvalue_ref(
             OutputInterface::Category::internal,
-            "core_sizes",
+            "core-sizes",
             get_core_sizes(*this)));
 
     // Save core diagnostic information

src/celeritas/optical/TrackInitParams.hh

@@ -33,6 +33,9 @@ class TrackInitParams final : public ParamsDataInterface<TrackInitParamsData>
     //! Access data on device
     DeviceRef const& device_ref() const final { return data_.device_ref(); }
 
+    //! Maximum number of initializers
+    size_type capacity() const { return host_ref().capacity; }
+
   private:
     // Host/device storage and reference
     CollectionMirror<TrackInitParamsData> data_;

src/celeritas/optical/detail/OpticalLaunchAction.cc

@@ -10,6 +10,8 @@
 #include "corecel/data/AuxParamsRegistry.hh"
 #include "corecel/data/AuxStateVec.hh"
 #include "corecel/io/Logger.hh"
+#include "corecel/io/OutputInterfaceAdapter.hh"
+#include "corecel/io/OutputRegistry.hh"
 #include "corecel/sys/ActionRegistry.hh"
 #include "celeritas/global/CoreParams.hh"
 #include "celeritas/global/CoreState.hh"
@@ -21,11 +23,36 @@
 #include "celeritas/track/TrackInitParams.hh"
 
 #include "OffloadParams.hh"
+#include "OpticalSizes.json.hh"
 
 namespace celeritas
 {
 namespace detail
 {
+namespace
+{
+//---------------------------------------------------------------------------//
+auto get_core_sizes(OpticalLaunchAction const& ola)
+{
+    // Optical core params
+    auto const& cp = ola.optical_params();
+
+    OpticalSizes result;
+    result.streams = cp.max_streams();
+
+    // NOTE: quantities are *per-process* quantities: integrated over streams,
+    // but not processes
+    result.generators = result.streams
+                        * ola.offload_params().host_ref().setup.capacity;
+    result.initializers = result.streams * cp.init()->capacity();
+    result.tracks = 0;
+
+    return result;
+}
+
+//---------------------------------------------------------------------------//
+}  // namespace
+
 //---------------------------------------------------------------------------//
 /*!
  * Construct and add to core params.
@@ -84,6 +111,13 @@ OpticalLaunchAction::OpticalLaunchAction(ActionId action_id,
         return inp;
     }());
 
+    // Add optical sizes
+    core.output_reg()->insert(
+        OutputInterfaceAdapter<detail::OpticalSizes>::from_rvalue_ref(
+            OutputInterface::Category::internal,
+            "optical-sizes",
+            get_core_sizes(*this)));
+
     // TODO: add generators to the *optical* stepping loop instead of part of
     // the main loop; for now just make sure enough track initializers are
     // allocated so that we can initialize them all at the beginning of step

src/celeritas/optical/detail/OpticalLaunchAction.hh

@@ -100,6 +100,22 @@ class OpticalLaunchAction : public AuxParamsInterface,
 
     // TODO: local end run to flush initializers??
 
+    //!@{
+    //! \name Accessors
+
+    //! Optical core params
+    optical::CoreParams const& optical_params() const
+    {
+        return *optical_params_;
+    }
+    //! Offload params
+    detail::OffloadParams const& offload_params() const
+    {
+        return *offload_params_;
+    }
+
+    //!@}
+
   private:
     using ActionGroupsT = ActionGroups<optical::CoreParams, optical::CoreState>;
     using SPOpticalParams = std::shared_ptr<optical::CoreParams>;

src/celeritas/optical/detail/OpticalSizes.json.hh

@@ -0,0 +1,46 @@
+//------------------------------- -*- C++ -*- -------------------------------//
+// Copyright Celeritas contributors: see top-level COPYRIGHT file for details
+// SPDX-License-Identifier: (Apache-2.0 OR MIT)
+//---------------------------------------------------------------------------//
+//! \file celeritas/optical/detail/OpticalSizes.json.hh
+//---------------------------------------------------------------------------//
+#pragma once
+
+#include <nlohmann/json.hpp>
+
+#include "corecel/Types.hh"
+#include "corecel/io/JsonUtils.json.hh"
+
+namespace celeritas
+{
+namespace detail
+{
+//---------------------------------------------------------------------------//
+/*!
+ * Save state size/capacity counters.
+ *
+ * These should be *integrated* across streams, *per process*.
+ */
+struct OpticalSizes
+{
+    size_type generators{};
+    size_type initializers{};
+    size_type tracks{};
+    size_type streams{};
+};
+
+//---------------------------------------------------------------------------//
+
+void to_json(nlohmann::json& j, OpticalSizes const& inp)
+{
+    j = {
+        CELER_JSON_PAIR(inp, generators),
+        CELER_JSON_PAIR(inp, initializers),
+        CELER_JSON_PAIR(inp, tracks),
+        CELER_JSON_PAIR(inp, streams),
+    };
+}
+
+//---------------------------------------------------------------------------//
+}  // namespace detail
+}  // namespace celeritas