extend tools documentation; add metadata diagramm by @7048730

Author: nuest

docs/03_system-scope-and-context.md

@@ -18,4 +18,4 @@ Communication partner | Exchanged data | Technology/protocol
 
 ### 3.2 Technical context
 
-All components use `HTTP` over cable networks connections for communication of all exchanged data (metadata documents, ERC, Linux containers, etc.).
+All components use `HTTP(S)` over cable networks connections for communication (metadata documents, ERC, Linux containers, etc.).

docs/04_solution-strategy.md

@@ -41,8 +41,8 @@ _The client application manages the control flow_ of all user interactions.
 
 ### Tools
 
-If standalone tools are developed, they should provide a command-line interface (CLI).
-The CLI allows integration into microservices when needed and to package tools including their dependencies as containers.
-Thanks to the container architecture and the controlled, we don't need to worry about documentation for or distribution/packaging of these tools.
-It must only be ensured they are correctly installed using the microservice's Dockerfile.
-The only required documentation is for the installation into a container and usage of the CLI.
+If standalone tools are developed, they provide a command-line interface (CLI).
+The CLI allows integration into microservices when needed and to package tools including their dependencies as containers and distributing them using a container registry.
+These _2nd level containers_ are started by the microservices and can run either next to the microservices or in an independent container cluster, providing scalability.
+It must only be ensured they are correctly configured in each microservice.
+The only required documentation is the installation into a container and usage of the CLI.

docs/05_building-block-view.md

@@ -148,7 +148,7 @@ The microservices all access one main database and a shared file storage.
 ##### 5.2.7.4 Blackbox Tools
 
 Some functionality is developed as standalone tools and used as such in the microservices instead of re-implementing features.
-These tools are integrated via their command line interface (CLI).
+These tools are integrated via their command line interface (CLI) and executed as _2nd level containers_ by microservices.
 
 ##### 5.2.7.5 Blackbox Databases
 
@@ -239,6 +239,23 @@ The search index is accessed by clients through the search endpoint provided by
 [meta](https://github.com/o2r-project/o2r-meta) | Python | scripts for extraction, translation and validation of metadata; for details see [metadata documentation](/metadata)
 [containerit](https://github.com/o2r-project/containerit) | R | generation of Dockerfiles based on R sessions and scripts
 
+Each tool's code repository includes one or more `Dockerfiles`, which are automatically build and published on Docker Hub.
+The microservices use the tool's Docker images to execute the tools instead of installing all their dependencies into the microservices.
+The advantages are a controlled environment for the tool usage, independent development cycles and updating of the tools, smaller independent images for the microservices, and scalability.
+
+#### Meta
+
+Meta provides a CLI for each step of the metadata processing required in the reproducibility service as shown by the following diagram.
+After each step the created metadata is saved as a file per model to a directory in the compendium.
+A detailed view of the meta tool usage in the creation process is provided in the runtime view [ERC Creation](#61-erc-creation).
+
+[![whitebox meta tool](img/5.2-whitebox-meta-tool.png)](img/5.2-whitebox-meta-tool.png)
+
+#### Containerit
+
+The containerit tool extracts required dependencies from ERC main documents and uses the information and external configuration to create a Dockerfile, which executes the full computational workflow when the container is started.
+Its main strategy is to analyse the session at the end of executing the full workflow.
+
 #### 5.3.4 Whitebox ephemeral file storage
 
 A host directory is mounted into every container to the location `/tmp/o2r`.

docs/img/5.2-whitebox-meta-tool.png

@@ -0,0 +1,170 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:java="http://www.yworks.com/xml/yfiles-common/1.0/java" xmlns:sys="http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0" xmlns:x="http://www.yworks.com/xml/yfiles-common/markup/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
+ <!--Created by yEd 3.17-->
+ <key attr.name="Beschreibung" attr.type="string" for="graph" id="d0"/>
+ <key for="port" id="d1" yfiles.type="portgraphics"/>
+ <key for="port" id="d2" yfiles.type="portgeometry"/>
+ <key for="port" id="d3" yfiles.type="portuserdata"/>
+ <key attr.name="url" attr.type="string" for="node" id="d4"/>
+ <key attr.name="description" attr.type="string" for="node" id="d5"/>
+ <key for="node" id="d6" yfiles.type="nodegraphics"/>
+ <key for="graphml" id="d7" yfiles.type="resources"/>
+ <key attr.name="url" attr.type="string" for="edge" id="d8"/>
+ <key attr.name="description" attr.type="string" for="edge" id="d9"/>
+ <key for="edge" id="d10" yfiles.type="edgegraphics"/>
+ <graph edgedefault="directed" id="G">
+ <data key="d0"/>
+ <node id="n0">
+ <data key="d6">
+ <y:ShapeNode>
+ <y:Geometry height="113.0" width="113.0" x="350.0" y="405.0"/>
+ <y:Fill hasColor="false" transparent="false"/>
+ <y:BorderStyle color="#000000" raised="false" type="line" width="1.0"/>
+ <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="31.9375" horizontalTextPosition="center" iconTextGap="4" modelName="custom" textColor="#000000" verticalTextPosition="bottom" visible="true" width="68.2421875" x="22.37890625" y="40.53125">User
+workspace<y:LabelModel>
+ <y:SmartNodeLabelModel distance="4.0"/>
+ </y:LabelModel>
+ <y:ModelParameter>
+ <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+ </y:ModelParameter>
+ </y:NodeLabel>
+ <y:Shape type="roundrectangle"/>
+ </y:ShapeNode>
+ </data>
+ </node>
+ <node id="n1">
+ <data key="d6">
+ <y:ShapeNode>
+ <y:Geometry height="113.0" width="113.0" x="601.5" y="405.0"/>
+ <y:Fill hasColor="false" transparent="false"/>
+ <y:BorderStyle color="#000000" raised="false" type="line" width="1.0"/>
+ <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="17.96875" horizontalTextPosition="center" iconTextGap="4" modelName="custom" textColor="#000000" verticalTextPosition="bottom" visible="true" width="90.14453125" x="11.427734375" y="47.515625">Raw Metadata<y:LabelModel>
+ <y:SmartNodeLabelModel distance="4.0"/>
+ </y:LabelModel>
+ <y:ModelParameter>
+ <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+ </y:ModelParameter>
+ </y:NodeLabel>
+ <y:Shape type="roundrectangle"/>
+ </y:ShapeNode>
+ </data>
+ </node>
+ <node id="n2">
+ <data key="d6">
+ <y:ShapeNode>
+ <y:Geometry height="113.0" width="113.0" x="853.0" y="405.0"/>
+ <y:Fill hasColor="false" transparent="false"/>
+ <y:BorderStyle color="#004286" raised="false" type="line" width="1.0"/>
+ <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="bold" hasBackgroundColor="false" hasLineColor="false" height="17.96875" horizontalTextPosition="center" iconTextGap="4" modelName="custom" textColor="#000000" verticalTextPosition="bottom" visible="true" width="95.125" x="8.9375" y="47.515625">o2r Metadata<y:LabelModel>
+ <y:SmartNodeLabelModel distance="4.0"/>
+ </y:LabelModel>
+ <y:ModelParameter>
+ <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+ </y:ModelParameter>
+ </y:NodeLabel>
+ <y:Shape type="roundrectangle"/>
+ </y:ShapeNode>
+ </data>
+ </node>
+ <node id="n3">
+ <data key="d6">
+ <y:ShapeNode>
+ <y:Geometry height="113.0" width="113.0" x="1104.5" y="405.0"/>
+ <y:Fill hasColor="false" transparent="false"/>
+ <y:BorderStyle color="#000000" raised="false" type="line" width="1.0"/>
+ <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="45.90625" horizontalTextPosition="center" iconTextGap="4" modelName="custom" textColor="#000000" verticalTextPosition="bottom" visible="true" width="68.7578125" x="22.12109375" y="33.546875">3rd party 
+repository 
+Metadata<y:LabelModel>
+ <y:SmartNodeLabelModel distance="4.0"/>
+ </y:LabelModel>
+ <y:ModelParameter>
+ <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+ </y:ModelParameter>
+ </y:NodeLabel>
+ <y:Shape type="roundrectangle"/>
+ </y:ShapeNode>
+ </data>
+ </node>
+ <edge id="e0" source="n0" target="n1">
+ <data key="d10">
+ <y:PolyLineEdge>
+ <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+ <y:LineStyle color="#000000" type="line" width="1.0"/>
+ <y:Arrows source="none" target="standard"/>
+ <y:EdgeLabel alignment="center" configuration="AutoFlippingLabel" distance="2.0" fontFamily="Dialog" fontSize="12" fontStyle="bold" hasBackgroundColor="false" hasLineColor="false" height="17.96875" horizontalTextPosition="center" iconTextGap="4" modelName="custom" preferredPlacement="anywhere" ratio="0.5" textColor="#000000" verticalTextPosition="bottom" visible="true" width="73.380859375" x="32.53961181640625" y="21.015625">extraction<y:LabelModel>
+ <y:SmartEdgeLabelModel autoRotationEnabled="false" defaultAngle="0.0" defaultDistance="10.0"/>
+ </y:LabelModel>
+ <y:ModelParameter>
+ <y:SmartEdgeLabelModelParameter angle="0.0" distance="30.0" distanceToCenter="true" position="right" ratio="0.5" segment="0"/>
+ </y:ModelParameter>
+ <y:PreferredPlacementDescriptor angle="0.0" angleOffsetOnRightSide="0" angleReference="absolute" angleRotationOnRightSide="co" distance="-1.0" frozen="true" placement="anywhere" side="anywhere" sideReference="relative_to_edge_flow"/>
+ </y:EdgeLabel>
+ <y:BendStyle smoothed="false"/>
+ </y:PolyLineEdge>
+ </data>
+ </edge>
+ <edge id="e1" source="n1" target="n2">
+ <data key="d10">
+ <y:PolyLineEdge>
+ <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+ <y:LineStyle color="#000000" type="line" width="1.0"/>
+ <y:Arrows source="none" target="standard"/>
+ <y:EdgeLabel alignment="center" configuration="AutoFlippingLabel" distance="2.0" fontFamily="Dialog" fontSize="12" fontStyle="bold" hasBackgroundColor="false" hasLineColor="false" height="31.9375" horizontalTextPosition="center" iconTextGap="4" modelName="custom" preferredPlacement="anywhere" ratio="0.5" textColor="#000000" verticalTextPosition="bottom" visible="true" width="103.69140625" x="17.38433837890625" y="14.03125">brokering with
+o2r mapping<y:LabelModel>
+ <y:SmartEdgeLabelModel autoRotationEnabled="false" defaultAngle="0.0" defaultDistance="10.0"/>
+ </y:LabelModel>
+ <y:ModelParameter>
+ <y:SmartEdgeLabelModelParameter angle="0.0" distance="30.0" distanceToCenter="true" position="right" ratio="0.5" segment="0"/>
+ </y:ModelParameter>
+ <y:PreferredPlacementDescriptor angle="0.0" angleOffsetOnRightSide="0" angleReference="absolute" angleRotationOnRightSide="co" distance="-1.0" frozen="true" placement="anywhere" side="anywhere" sideReference="relative_to_edge_flow"/>
+ </y:EdgeLabel>
+ <y:BendStyle smoothed="false"/>
+ </y:PolyLineEdge>
+ </data>
+ </edge>
+ <edge id="e2" source="n2" target="n3">
+ <data key="d10">
+ <y:PolyLineEdge>
+ <y:Path sx="-30.0" sy="0.0" tx="0.0" ty="0.0"/>
+ <y:LineStyle color="#000000" type="line" width="1.0"/>
+ <y:Arrows source="none" target="standard"/>
+ <y:EdgeLabel alignment="center" configuration="AutoFlippingLabel" distance="2.0" fontFamily="Dialog" fontSize="12" fontStyle="bold" hasBackgroundColor="false" hasLineColor="false" height="45.90625" horizontalTextPosition="center" iconTextGap="4" modelName="custom" preferredPlacement="anywhere" ratio="0.5" textColor="#000000" verticalTextPosition="bottom" visible="true" width="103.69140625" x="17.41314697265625" y="7.046875">brokering with
+repository
+mapping<y:LabelModel>
+ <y:SmartEdgeLabelModel autoRotationEnabled="false" defaultAngle="0.0" defaultDistance="10.0"/>
+ </y:LabelModel>
+ <y:ModelParameter>
+ <y:SmartEdgeLabelModelParameter angle="0.0" distance="30.0" distanceToCenter="true" position="right" ratio="0.5" segment="0"/>
+ </y:ModelParameter>
+ <y:PreferredPlacementDescriptor angle="0.0" angleOffsetOnRightSide="0" angleReference="absolute" angleRotationOnRightSide="co" distance="-1.0" frozen="true" placement="anywhere" side="anywhere" sideReference="relative_to_edge_flow"/>
+ </y:EdgeLabel>
+ <y:BendStyle smoothed="false"/>
+ </y:PolyLineEdge>
+ </data>
+ </edge>
+ <edge id="e3" source="n2" target="n2">
+ <data key="d10">
+ <y:PolyLineEdge>
+ <y:Path sx="37.75" sy="0.0" tx="-29.5" ty="-41.5">
+ <y:Point x="947.25" y="358.5"/>
+ <y:Point x="880.0" y="358.5"/>
+ </y:Path>
+ <y:LineStyle color="#000000" type="line" width="1.0"/>
+ <y:Arrows source="none" target="standard"/>
+ <y:EdgeLabel alignment="center" configuration="AutoFlippingLabel" distance="2.0" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="17.96875" horizontalTextPosition="center" iconTextGap="4" modelName="custom" preferredPlacement="anywhere" ratio="0.5" textColor="#000000" verticalTextPosition="bottom" visible="true" width="66.23828125" x="-66.744140625" y="-85.480224609375">User Input<y:LabelModel>
+ <y:SmartEdgeLabelModel autoRotationEnabled="false" defaultAngle="0.0" defaultDistance="10.0"/>
+ </y:LabelModel>
+ <y:ModelParameter>
+ <y:SmartEdgeLabelModelParameter angle="0.0" distance="30.0" distanceToCenter="true" position="right" ratio="0.5" segment="1"/>
+ </y:ModelParameter>
+ <y:PreferredPlacementDescriptor angle="0.0" angleOffsetOnRightSide="0" angleReference="absolute" angleRotationOnRightSide="co" distance="-1.0" frozen="true" placement="anywhere" side="anywhere" sideReference="relative_to_edge_flow"/>
+ </y:EdgeLabel>
+ <y:BendStyle smoothed="false"/>
+ </y:PolyLineEdge>
+ </data>
+ </edge>
+ </graph>
+ <data key="d7">
+ <y:Resources/>
+ </data>
+</graphml>