started run_step.sh and corresponding files

INSATLL.md

-PWD=`pwd`
-export PATH=$PATH:${PWD}/bin
\ No newline at end of file
+1. install docker, minikube
+
+2. mkdir pipeline
+3. mkdir images
+
+4. mkdir archives-pv (and populate it with the archive you need: step-name/all you need)
+5. mkdir artifacts-pv
+6. mkdir runtime-parameters-pv
+
+6. start minikube: minikube start
+
+7. mount permanent volume (in other terminal)
+   minikube mount archives-pv:/mnt/archives-pv
+   minikube mount artifacts-pv:/mnt/artifacts-pv
+   minikube mount runtime-parameters-pv:/mnt/parameters-pv
+
+8. Create the permanent volume
+   kubectl apply -f create-archives-pv.yaml
+   kubectl apply -f create-artifacts-pv.yaml
+   kubectl apply -f create-parameters-pv.yaml
+
+9. set config.toml as wanted
+
+10. setup env
+    PWD=`pwd`
+    export PATH=$PATH:${PWD}/bin
+
+11. setup-pipeline.sh (uses config.toml)
+
+12. build_pipeline.sh (comming soon)
+
+13. run_pipeline.sh (comming soon)
+
+14. enjoy the results in artifacts-pv !!
+
+
+

bin/run_step.sh

+STEP_NAME=$1
+COMMIT=$2
+
+source pipe-env.sh
+
+# SETUP TMP DIR
+TMP_HASH=$(date | sha1sum)
+TMP_HASH=${TMP_HASH%% *}
+
+echo ${TMP_HASH}
+
+mkdir ${PIPE_ROOT}/${ARTIFACTS}/.${STEP_NAME}_${TMP_HASH}
+
+TMP_DIR=${PIPE_ROOT}/${ARTIFACTS}/.${STEP_NAME}_${TMP_HASH}
+touch ${TMP_DIR}/trace.toml
+echo "[${STEP_NAME}]" >> ${TMP_DIR}/trace.toml
+touch ${TMP_DIR}/parameters_checksum.sha1
+
+
+# BUILD HASH
+BUILD_HASH=$(docker inspect ${STEP_NAME}:${COMMIT} --format="{{.Id}}")
+echo "build_hash = \"${BUILD_HASH}\"" >> ${TMP_DIR}/trace.toml
+
+# PARAM HASH (only necessary if Parameters not in GIT repo --> COMMIT)
+cd ${PIPE_ROOT}/${RUNTIME_PARAMS}
+filenames=$(find "${STEP_NAME}" -type f)
+
+for filename in ${filenames};
+do
+    sha1sum ${filename} >> ${TMP_DIR}/parameters_checksum.sha1
+done
+
+cd ${PIPE_ROOT}
+PARAM_HASH=$(sha1sum ${TMP_DIR}/parameters_checksum.sha1)
+PARAM_HASH=${PARAM_HASH%% *}
+echo "parameters_hash = \"sha1:${PARAM_HASH}\"" >> ${TMP_DIR}/trace.toml
+
+# ARCHIVE HASH (only necessary if ArchDescr not in GIT repo --> COMMIT)
+STEPS=`./stoml ${PIPE_ROOT}/config.toml steps`
+
+echo $STEPS
+
+if [[ ! " ${STEPS[*]} " =~ " ${STEP_NAME} " ]]; then
+    echo "ERROR: the step: ${STEP_NAME} is not defined."
+    exit 1
+fi
+
+ARCHIVE_DEPS=(`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.archives`)
+
+for archive in "${ARCHIVE_DEPS[@]}";
+do
+    cd ${PIPE_ROOT}/${ARCHIVES}
+
+    filenames=$(find "${archive}" -type f)
+
+    for filename in ${filenames};
+    do
+	sha1sum ${filename} >> ${TMP_DIR}/archives_checksum.sha1
+    done
+
+    cd ${PIPE_ROOT}
+
+done;
+
+ARCHIVES_HASH=$(sha1sum ${TMP_DIR}/archives_checksum.sha1)
+ARCHIVES_HASH=${ARCHIVES_HASH%% *}
+echo "archives_hash = \"sha1:${ARCHIVES_HASH}\"" >> ${TMP_DIR}/trace.toml
+
+# CREATE STEP HASH
+
+STEP_HASH_ID="${BUILD_HASH}-${PARAM_HASH}-${ARCHIVES_HASH}"
+STEP_HASH=$(echo "${STEP_HASH}" | sha1sum)
+
+echo ${STEP_HASH_ID}
+echo ${STEP_HASH}
+
+echo "step-id = \"${STEP_HASH_ID}\"" >> ${TMP_DIR}/trace.toml
+echo "step-hash = \"sha1:${STEP_HASH}\"" >> ${TMP_DIR}/trace.toml
+
+exit 0
+
+# CREATE DEPS HASH
+
+STEPS=`./stoml ${PIPE_ROOT}/config.toml steps`
+
+echo $STEPS
+
+if [[ ! " ${STEPS[*]} " =~ " ${STEP_NAME} " ]]; then
+    echo "ERROR: the step: ${STEP_NAME} is not defined."
+    exit 1
+fi
+
+ARTIFACTS_DEPS=(`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.artifacts`)
+
+for archive in "${ARTIFACTS_DEPS[@]}";
+do
+    cd ${PIPE_ROOT}/${ARCHIVES}
+
+    filenames=$(find "${archive}" -type f)
+
+    for filename in ${filenames};
+    do
+	sha1sum ${filename} >> ${TMP_DIR}/archives_checksum.sha1
+    done
+
+    cd ${PIPE_ROOT}
+
+done;
+
+exit 0
+
+#PARAM_HASH=
+

bin/setup_pipeline.sh

@@ -2,6 +2,7 @@
 
 PIPE_ROOT=`pwd`
 
+source ${PIPE_ROOT}/pipe-env.sh
 source ${PIPE_ROOT}/src/setup.sh
 
 setup_pipeline

bin/setup_step.sh

@@ -34,6 +34,7 @@ fi;
 
 PIPE_ROOT=`pwd`
 
+source ${PIPE_ROOT}/pipe-env.sh
 source ${PIPE_ROOT}/src/setup.sh
 
 setup_step ${PIPE_ROOT} ${STEP_NAME}

config.toml

@@ -8,6 +8,8 @@ type = "git_repo"
 repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features.git"
 commit = "1ccb3969"
 
+archives = ["ml_solar_plates_mock"]
+
 [steps.label-features]
 
 type = "git_repo"

pipe-env.sh

 export PIPE_ROOT=`pwd`
 export IMAGES_ARCH="images"
 export PIPELINE="pipeline"
+export RUNTIME_PARAMS="runtime-parameters-pv"
+export ARTIFACTS="artifacts-pv"
+export ARCHIVES="archives-pv"

src/setup.sh

@@ -105,6 +105,7 @@ function setup_step() {
 	    local REPO_URL=`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.repo_url`
 	    local COMMIT=`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.commit`
 
+	    # setup the step
 	    setup_step_from_git_repo ${PIPE_ROOT} ${STEP_NAME} ${REPO_URL} ${COMMIT}
 
 	    if [[ $? != 0 ]];
@@ -112,6 +113,19 @@ function setup_step() {
 		return $?
 	    fi;
 
+	    # create a runtime-parameters for the step
+	    if [ ! -d ${PIPE_ROOT}/${RUNTIME_PARAMS}/${STEP_NAME} ];
+	    then
+		mkdir ${PIPE_ROOT}/${RUNTIME_PARAMS}/${STEP_NAME}
+
+		if [[ $? != 0 ]];
+		then
+		    echo "ERROR: Could not create the runtime-parameters folder. Do you have the write-rights there: ${PIPE_ROOT}/${RUNTIME_PARAM} ?"
+		    return $?
+		fi;
+		
+	    fi;
+	    
 	    ;;
 
 	*)