Remote subdir and aero DVs to sens file fix

funtofem/driver/funtofem_shape_driver.py

@@ -200,7 +200,7 @@ def __init__(
 
         if not self.is_remote:
             assert solvers.flow is not None
-            assert solvers.structural is not None
+            assert solvers.structural is not None or model.structural is not None
         self._first_forward = True
 
         # initialize adjoint state variables to zero for writing sens files
@@ -352,7 +352,7 @@ def solve_forward(self):
             # write the funtofem design input file
             self.model.write_design_variables_file(
                 self.comm,
-                filename=Remote.paths(self.remote.main_dir).design_file,
+                filename=Remote.paths(self.comm, self.remote.main_dir).design_file,
                 root=0,
             )
 
@@ -378,7 +378,7 @@ def solve_forward(self):
                 # read in the funtofem design input file
                 self.model.read_design_variables_file(
                     self.comm,
-                    filename=Remote.paths(self.flow_dir).design_file,
+                    filename=Remote.paths(self.comm, self.flow_dir).design_file,
                     root=0,
                 )
 
@@ -390,13 +390,14 @@ def solve_forward(self):
             if not self.is_paired:
                 filepath = self.flow_aim.sens_file_path
             else:
-                filepath = Remote.paths(self.flow_dir).aero_sens_file
+                filepath = Remote.paths(self.comm, self.flow_dir).aero_sens_file
 
             # write the sensitivity file for the FUN3D AIM
             self.model.write_sensitivity_file(
                 comm=self.comm,
                 filename=filepath,
                 discipline="aerodynamic",
+                write_dvs=False,
             )
 
         # post analysis for FUN3D mesh morphing
@@ -434,8 +435,10 @@ def solve_adjoint(self):
             if self.is_paired:
                 write_struct = True
                 write_aero = True
-                struct_sensfile = Remote.paths(self.flow_dir).struct_sens_file
-                aero_sensfile = Remote.paths(self.flow_dir).aero_sens_file
+                struct_sensfile = Remote.paths(
+                    self.comm, self.flow_dir
+                ).struct_sens_file
+                aero_sensfile = Remote.paths(self.comm, self.flow_dir).aero_sens_file
             else:
                 if self.struct_shape:
                     write_struct = True
@@ -463,6 +466,7 @@ def solve_adjoint(self):
                     comm=self.comm,
                     filename=aero_sensfile,
                     discipline="aerodynamic",
+                    write_dvs=False,
                 )
 
         if self.struct_shape:  # either remote or regular

funtofem/driver/oneway_aero_driver.py

@@ -261,7 +261,7 @@ def solve_forward(self):
             # write the funtofem design input file
             self.model.write_design_variables_file(
                 self.comm,
-                filename=Remote.paths(self.remote.main_dir).design_file,
+                filename=Remote.paths(self.comm, self.remote.main_dir).design_file,
                 root=0,
             )
 
@@ -279,7 +279,7 @@ def solve_forward(self):
             if self.is_paired:
                 self.model.read_design_variables_file(
                     self.comm,
-                    filename=Remote.paths(self.flow_dir).design_file,
+                    filename=Remote.paths(self.comm, self.flow_dir).design_file,
                     root=0,
                 )
 
@@ -299,13 +299,14 @@ def solve_forward(self):
             if not self.is_paired:
                 filepath = self.flow_aim.sens_file_path
             else:
-                filepath = Remote.paths(self.flow_dir).aero_sens_file
+                filepath = Remote.paths(self.comm, self.flow_dir).aero_sens_file
 
             # write the sensitivity file for the FUN3D AIM
             self.model.write_sensitivity_file(
                 comm=self.comm,
                 filename=filepath,
                 discipline="aerodynamic",
+                write_dvs=False,
             )
 
         # post analysis for FUN3D mesh morphing
@@ -359,13 +360,14 @@ def solve_adjoint(self):
             if not self.is_paired:
                 filepath = self.flow_aim.sens_file_path
             else:
-                filepath = Remote.paths(self.flow_dir).aero_sens_file
+                filepath = Remote.paths(self.comm, self.flow_dir).aero_sens_file
 
             # write the sensitivity file for the FUN3D AIM
             self.model.write_sensitivity_file(
                 comm=self.comm,
                 filename=filepath,
                 discipline="aerodynamic",
+                write_dvs=False,
             )
 
         # shape derivative section

funtofem/driver/oneway_struct_driver.py

@@ -161,7 +161,7 @@ def uses_tacs(self) -> bool:
     def analysis_sens_file(self):
         """write location of sens file when used in FuntofemShapeDriver for double oneway drivers (analysis version)"""
         if self.fun3d_dir is None:
-            return Remote.paths(self.fun3d_dir).struct_sens_file
+            return Remote.paths(self.comm, self.fun3d_dir).struct_sens_file
         else:
             return None
 

funtofem/interface/utils/remote.py

@@ -40,14 +40,21 @@ def __init__(
         self.struct_name = struct_name
 
     @classmethod
-    def paths(cls, main_dir, aero_name="fun3d", struct_name="tacs"):
+    def paths(cls, comm, main_dir, aero_name="fun3d", struct_name="tacs"):
         return cls(
             analysis_file=None,
-            main_dir=main_dir,
+            main_dir=cls.remote_dir(comm, main_dir),
             aero_name=aero_name,
             struct_name=struct_name,
         )
 
+    @classmethod
+    def remote_dir(cls, comm, main_dir):
+        _remote_dir = os.path.join(main_dir, "remote")
+        if comm.rank == 0 and not (os.path.exists(_remote_dir)):
+            os.mkdir(_remote_dir)
+        return _remote_dir
+
     @classmethod
     def fun3d_path(cls, main_dir, filename):
         return os.path.join(main_dir, filename)

funtofem/model/funtofem_model.py

@@ -633,7 +633,9 @@ def read_aero_loads(self, comm, filename, root=0):
         # return the loads data
         return loads_data
 
-    def write_sensitivity_file(self, comm, filename, discipline="aerodynamic", root=0):
+    def write_sensitivity_file(
+        self, comm, filename, discipline="aerodynamic", root=0, write_dvs: bool = True
+    ):
         """
         Write the sensitivity file.
 
@@ -656,6 +658,8 @@ def write_sensitivity_file(self, comm, filename, discipline="aerodynamic", root=
             The name of the discipline sensitivity data to be written
         root: int
             The rank of the processor that will write the file
+        write_dvs: bool
+            whether to write the design variables for this discipline
         """
 
         funcs = self.get_functions()
@@ -674,10 +678,11 @@ def write_sensitivity_file(self, comm, filename, discipline="aerodynamic", root=
         if comm.rank == root:
             variables = self.get_variables()
             discpline_vars = []
-            for var in variables:
-                # Write the variables whose analysis_type matches the discipline string.
-                if discipline == var.analysis_type:
-                    discpline_vars.append(var)
+            if write_dvs:  # flag for registration then writing out discipline dvs
+                for var in variables:
+                    # Write the variables whose analysis_type matches the discipline string.
+                    if discipline == var.analysis_type:
+                        discpline_vars.append(var)
 
             # Write out the number of sets of discpline variables
             num_dvs = len(discpline_vars)

include/TransferScheme.h

@@ -32,23 +32,28 @@ inline double F2FImagPart(const F2FComplex &c) { return imag(c); }
 inline double F2FRealPart(const double &r) { return r; }
 
 // Compute the absolute value
-inline F2FReal F2Ffabs(const F2FReal &c) {
-  if (c < 0.0) {
+inline F2FReal F2Ffabs(const F2FReal &c)
+{
+  if (c < 0.0)
+  {
     return -c;
   }
   return c;
 }
 
 // Compute the absolute value
-inline F2FComplex F2Ffabs(const F2FComplex &c) {
-  if (real(c) < 0.0) {
+inline F2FComplex F2Ffabs(const F2FComplex &c)
+{
+  if (real(c) < 0.0)
+  {
     return -c;
   }
   return c;
 }
 
-class TransferScheme {
- public:
+class TransferScheme
+{
+public:
   TransferScheme(MPI_Comm global_comm, MPI_Comm struct_comm, int struct_root,
                  MPI_Comm aero_comm, int aero_root, int struct_node_dof = 3,
                  int aero_node_dof = 3)
@@ -58,16 +63,17 @@ class TransferScheme {
         aero_comm(aero_comm),
         aero_root(aero_root),
         struct_node_dof(struct_node_dof),
-        aero_node_dof(aero_node_dof) {
+        aero_node_dof(aero_node_dof)
+  {
     na = 0;
     na_global = 0;
     ns = 0;
     ns_local = 0;
     mesh_update = 0;
 
-    Xa = NULL;        // Local array of aerodynamic nodes
-    Xs = NULL;        // Global array of structural nodes
-    Xs_local = NULL;  // Local array of structural nodes
+    Xa = NULL;       // Local array of aerodynamic nodes
+    Xs = NULL;       // Global array of structural nodes
+    Xs_local = NULL; // Local array of structural nodes
 
     object_id = object_count;
     object_count++;
@@ -92,7 +98,7 @@ class TransferScheme {
   int getLocalAeroArrayLen() { return aero_node_dof * na; }
   int getLocalStructArrayLen() { return struct_node_dof * ns_local; }
 
- protected:
+protected:
   // Distribute the structural mesh if mesh_update is true on one of the
   // processors.
   void distributeStructuralMesh();
@@ -124,21 +130,21 @@ class TransferScheme {
                       F2FScalar *W, double tol = 1e-7);
 
   // Communicators
-  MPI_Comm global_comm;  // Global communicator
-  MPI_Comm struct_comm;  // Communicator for the structures
-  int struct_root;       // Structural rank-0 proc on global_comm
-  MPI_Comm aero_comm;    // Communicator for the aerodynamics
-  int aero_root;         // Aerodynamic rank-0 proc on global_comm
-  int struct_node_dof;   // Degrees of freedom per structural node
-  int aero_node_dof;     // Degrees of freedom per aerodynamic node
+  MPI_Comm global_comm; // Global communicator
+  MPI_Comm struct_comm; // Communicator for the structures
+  int struct_root;      // Structural rank-0 proc on global_comm
+  MPI_Comm aero_comm;   // Communicator for the aerodynamics
+  int aero_root;        // Aerodynamic rank-0 proc on global_comm
+  int struct_node_dof;  // Degrees of freedom per structural node
+  int aero_node_dof;    // Degrees of freedom per aerodynamic node
 
   // Keep track if the mesh has been updated
   int mesh_update;
 
   // Aerodynamic data
-  F2FScalar *Xa;  // Aerodynamics node locations (x, y, z) at each node
-  int na;         // Number of local aerodynamic nodes
-  int na_global;  // Number of global aerodynamic nodes (on all aero procs)
+  F2FScalar *Xa; // Aerodynamics node locations (x, y, z) at each node
+  int na;        // Number of local aerodynamic nodes
+  int na_global; // Number of global aerodynamic nodes (on all aero procs)
 
   // Structural data
   // Degrees of freedom per node for the structural solution and load vector.
@@ -149,30 +155,35 @@ class TransferScheme {
   // however, if the structures uses u, v, w, theta_x, theta_y, theta_z, then
   // dof_per_node = 6. Handling this case is up to the specific transfer
   // scheme implementation.
-  F2FScalar *Xs;        // Global array of (x, y,z) locations for structures
-  F2FScalar *Xs_local;  // Local array of (x, y, z) locations for structures
-  int ns;        // Number of global structural nodes across all struct procs
-  int ns_local;  // Number of local structural nodes on this processor
+  F2FScalar *Xs;       // Global array of (x, y,z) locations for structures
+  F2FScalar *Xs_local; // Local array of (x, y, z) locations for structures
+  int ns;              // Number of global structural nodes across all struct procs
+  int ns_local;        // Number of local structural nodes on this processor
 
   // Transfer scheme object counter and ID
   static int object_count;
   int object_id;
 };
 
-class LDTransferScheme : public TransferScheme {
- public:
+class LDTransferScheme : public TransferScheme
+{
+public:
   LDTransferScheme(MPI_Comm global_comm, MPI_Comm struct_comm, int struct_root,
                    MPI_Comm aero_comm, int aero_root, int struct_node_dof = 3)
       : TransferScheme(global_comm, struct_comm, struct_root, aero_comm,
-                       aero_root, struct_node_dof, 3) {
+                       aero_root, struct_node_dof, 3)
+  {
     Us = NULL;
     Fa = NULL;
   }
-  virtual ~LDTransferScheme() {
-    if (Us) {
+  virtual ~LDTransferScheme()
+  {
+    if (Us)
+    {
       delete[] Us;
     }
-    if (Fa) {
+    if (Fa)
+    {
       delete[] Fa;
     }
   }
@@ -192,10 +203,12 @@ class LDTransferScheme : public TransferScheme {
   // Action of Jacobians. These are valid when the transfer scheme is derived
   // using the method of virtual work. If not, they must be implemented
   // directly.
-  virtual void applydLdfA(const F2FScalar *vecs, F2FScalar *prods) {
+  virtual void applydLdfA(const F2FScalar *vecs, F2FScalar *prods)
+  {
     applydDduSTrans(vecs, prods);
   }
-  virtual void applydLdfATrans(const F2FScalar *vecs, F2FScalar *prods) {
+  virtual void applydLdfATrans(const F2FScalar *vecs, F2FScalar *prods)
+  {
     applydDduS(vecs, prods);
   }
 
@@ -248,7 +261,7 @@ class LDTransferScheme : public TransferScheme {
                          const F2FScalar *test_vec_s2, const F2FScalar h,
                          const double rtol, const double atol);
 
- protected:
+protected:
   // Aerodynamic load data
   F2FScalar *Fa;
 
@@ -269,20 +282,25 @@ class LDTransferScheme : public TransferScheme {
   void assembleM1(const F2FScalar *R, const F2FScalar *S, F2FScalar *A);
 };
 
-class ThermalTransfer : public TransferScheme {
- public:
+class ThermalTransfer : public TransferScheme
+{
+public:
   ThermalTransfer(MPI_Comm global_comm, MPI_Comm struct_comm, int struct_root,
                   MPI_Comm aero_comm, int aero_root)
       : TransferScheme(global_comm, struct_comm, struct_root, aero_comm,
-                       aero_root, 1, 1) {
+                       aero_root, 1, 1)
+  {
     Ha = NULL;
     Ts = NULL;
   }
-  virtual ~ThermalTransfer() {
-    if (Ha) {
+  virtual ~ThermalTransfer()
+  {
+    if (Ha)
+    {
       delete[] Ha;
     }
-    if (Ts) {
+    if (Ts)
+    {
       delete[] Ts;
     }
   }
@@ -316,7 +334,7 @@ class ThermalTransfer : public TransferScheme {
                              const F2FScalar *test_vec_s2, const F2FScalar h,
                              const double rtol, const double atol);
 
- protected:
+protected:
   // Aerodynamic load data
   F2FScalar *Ha;
 
@@ -333,4 +351,4 @@ F2FScalar vec_mag(const F2FScalar *x);
 F2FScalar vec_dot(const F2FScalar *x, const F2FScalar *y);
 F2FScalar det(const F2FScalar *A);
 
-#endif  // TRANSFER_SCHEME_H
+#endif // TRANSFER_SCHEME_H