almost done with run_step

INSATLL.md

@@ -24,6 +24,7 @@
 10. setup env
     PWD=`pwd`
     export PATH=$PATH:${PWD}/bin
+    export PATH=$PATH:${PWD}/manifests-template/bin
 
 11. setup-pipeline.sh (uses config.toml)
 

archives-pv.yaml

+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: ml-plate-archives
+spec:
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 10Gi
+  claimRef:
+    name: ml-solar-plates-archives-pv-claim
+  hostPath:
+    path: /mnt/ml_plates_archives
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: standard

artifacts-pv.yaml

+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: ml-plate-artifacts
+spec:
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 10Gi
+  claimRef:
+    name: ml-solar-plates-artifacts-pv-claim
+  hostPath:
+    path: /mnt/ml_plates_artifacts
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: standard

bin/run_step.sh

@@ -22,6 +22,19 @@ if [[ ! " ${STEPS[*]} " =~ " ${STEP_NAME} " ]]; then
     exit 1
 fi
 
+# Check dependancies
+if [ ! -z ${DEPS_DIR} ]
+then
+   for dep_dir in ${DEPS_DIR[@]};
+   do
+       if [ ! -d ${PIPE_ROOT}/${ARTIFACTS}/${dep_dir} ]
+       then
+	   echo "Error dependance does not exists!"
+	   exit
+       fi
+   done
+fi
+
 # setup tmp folder
 TMP_HASH=$(gen_hash_word)
 TMP_HASH=$(get_hash_from_word ${TMP_HASH})
@@ -107,9 +120,45 @@ fi;
 
 
 # run the step if it does not exist
-ARTIFACT_HASH=$(gen_hash_word --seed ${STEP_DIR_NAME})
-# ARTIFACT_HASH=$(gen_hash_word)
-echo "artifact_hash = \"${ARTIFACT_HASH}\"" >> ${TMP_DIR}/trace.toml
+
+# generate the step manifest
+if [ -z ${DEPS_DIR} ];
+then
+    gen-manifest.py ${STEP_NAME} ${COMMIT} .${STEP_NAME}_${TMP_HASH} --template templates/default.yaml --config config.toml --output ${TMP_DIR}/manifest.yaml
+else
+    gen-manifest.py ${STEP_NAME} ${COMMIT} .${STEP_NAME}_${TMP_HASH} --deps-dir ${DEPS_DIR[@]} --template templates/default.yaml --config config.toml --output ${TMP_DIR}/manifest.yaml
+fi;
+
+if [[ $? != 0 ]]; then
+    echo "Could not generate the manifest. Please check the template."
+    exit 1
+fi;
+
+# make sure the files are accessible via the docker group
+chown -R fournier:docker artifacts-pv
+chown -R fournier:docker archives-pv
+
+# apply the step manifest
+kubectl apply -f ${TMP_DIR}/manifest.yaml
+
+# waiting
+echo
+echo -n "WAITING for Job ${STEP_NAME} to finish..."
+sleep 2;
+until [ -z `kubectl get jobs ${STEP_NAME} -o jsonpath='{.status.active}'` ] ; do sleep 1; echo -n "." ; done
+
+status=$(kubectl get jobs ${STEP_NAME} -o=jsonpath='{.status.conditions[].type}')
+
+echo
+echo "Job ${STEP_NAME} finished with status: ${status}"
+echo
+
+if [[ "${status}" != "Complete" ]]; then
+    echo "Job ${STEP_NAME} has failed. Job was not deleted."
+    echo "To remove it, please use:"
+    echo "kubectl delete -f ${TMP_DIR}/manifest.yaml"
+    exit 1
+fi;
 
 # add the dependancies to the trace
 cat ${TMP_DIR}/trace_deps.toml >> ${TMP_DIR}/trace.toml
@@ -122,6 +171,13 @@ cp -r ${PIPE_ROOT}/${RUNTIME_PARAMS}/${STEP_NAME} ${TMP_DIR}/runtime-parameters
 mkdir ${TMP_DIR}/docker-image
 cp ${PIPE_ROOT}/${IMAGES_ARCH}/${STEP_NAME}_${COMMIT}.tar ${TMP_DIR}/docker-image/
 
+# gen the Artifact hash (incl. trace.toml)
+gen_dir_checksum ${TMP_DIR} ${TMP_DIR}/artifact_checksum.sha1 "True"
+if [[ $? != 0 ]]; then echo "Error in making checksum of the Artifacts"; exit 1; fi;
+
+ARTIFACT_HASH=$(gen_hash_from_checksum ${TMP_DIR}/artifact_checksum.sha1)
+echo ${ARTIFACT_HASH} > ${TMP_DIR}/hash.sha1
+
 # rename the folder with the artifact short-hash
 short_deps_hash=$(get_short_hash_from_word ${DEPS_HASH})
 short_step_hash=$(get_short_hash_from_word ${STEP_HASH})
@@ -132,4 +188,12 @@ mv ${TMP_DIR} ${PIPE_ROOT}/${ARTIFACTS}/${STEP_DIR_NAME}
 
 echo "${STEP_DIR_NAME}"
 
+read -p "Do you want to remove the Job? (y/n)" resp
+if [[ "${resp}" == "y" ]]; then
+    kubectl delete -f ${PIPE_ROOT}/${ARTIFACTS}/${STEP_DIR_NAME}/manifest.yaml
+else
+    echo "To remove it, please use:"
+    echo "kubectl delete -f ${PIPE_ROOT}/${ARTIFACTS}/${STEP_DIR_NAME}/manifest.yaml"
+fi;
+
 exit 0

config.toml

@@ -8,11 +8,11 @@ pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-
 
     [resources.archives_volume]
 
-         storage = "10Gi"
+         storage = "2Gi"
 
     [resources.artifacts_volume]
 
-         storage = "10Gi"
+         storage = "2Gi"
 
 
 # STEPS: define the configuration of the steps: repo, commit, volumes mount point...
@@ -35,7 +35,7 @@ pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-
 	# The list of archives and where they will be mounted in the container
         [[steps.extract-features.archives_volume]]
             name = "ml_solar_plates_mock"
-            mount_path = "/home/fournier/ml_solar_plates"
+            mount_path = "/home/fournier/solar_plates"
 
 
     [steps.label-features]
@@ -64,7 +64,7 @@ pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-
 
         type = "git_repo"
         repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/select-meaningful-plates-and-features.git"
-        commit = "81ca31c"
+        commit = "1fb3e7c16dcd254b31a1edbd6bff542b3382ab5b"
 
 	# The archives to be mounted
         archives = ["ml_solar_plates_mock"]
@@ -79,7 +79,7 @@ pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-
 	# The list of archives and where they will be mounted in the container
         [[steps.select-meaningful-plates-and-features.archives_volume]]
             name = "ml_solar_plates_mock"
-            mount_path = "home/fournier/ml_solar_plates"
+            mount_path = "home/fournier/solar_plates"
 
 	# The list of dependancies and where they are mounted
         [[steps.select-meaningful-plates-and-features.deps_volume]]

manifests-template/bin/gen-manifest.py

+#!/bin/env python
+
+from jinja2 import Template
+import toml
+
+# INPUT PARAM ARE:
+# STEP_NAME, COMMTI, DEPS_DIR (alike for run_step.sh)
+
+import argparse
+
+parser = argparse.ArgumentParser(description='Populate a step manifest template with pipeline config')
+parser.add_argument('step_name', metavar='STEP_NAME', type=str,
+                    help='The name of the step to run')
+parser.add_argument('commit', metavar='COMMIT', type=str,
+                    help='The commit of the src repo (tag of the corresponding image)')
+parser.add_argument('artifact_dir', metavar='ARTIFACT_DIR', type=str,
+                    help='The artifact step directory')
+parser.add_argument('--deps-dir', metavar='DEPS_DIR', type=str, nargs='+',
+                    default=[], help='The list of dependances directory')
+parser.add_argument('--template', metavar='TEMPLATE', type=str,
+                    default=None, help='The template yaml file')
+parser.add_argument('--config', metavar='CONFIG', type=str,
+                    default=None, help='The config toml file')
+parser.add_argument('--output', metavar='MANIFEST', type=str,
+                    default=None, help='The output yaml file')
+
+
+args = parser.parse_args()
+
+if args.template is None:
+    template_filename = "templates/default.yaml"
+else:
+    template_filename = args.template
+
+
+if args.config is None:
+    config_filename = "config.toml"
+else:
+    config_filename = args.config
+    
+with open(template_filename, 'r') as fd:
+    template = Template(fd.read())
+
+with open(config_filename, 'r') as fd:
+    config = toml.load(fd)
+
+# update config with
+
+step_config = config['steps'][args.step_name]
+
+step_config['step_name'] = args.step_name
+step_config['commit'] = args.commit
+step_config['project_name'] = config['project_name']
+step_config['resources'] = config['resources']
+step_config['artifact_volume']['sub_path'] = args.artifact_dir
+
+for dep_dir in args.deps_dir:
+
+    for volume in step_config['deps_volume']:
+        if volume['name'] == dep_dir[0:-27]: # name of the step
+            volume['sub_path'] = dep_dir
+
+for volume in step_config.get('archives_volume', []):
+    volume['sub_path'] = volume['name']
+    
+rendered = template.render(**step_config)
+
+if args.output is not None:
+    with open(args.output, 'w') as fd:
+        fd.write(rendered)
+else:
+    print(rendered)
+
+# sources:
+# https://stackoverflow.com/a/50557523
+# https://stackoverflow.com/a/50333856

manifests-template/multi-manifest.yaml

@@ -3,7 +3,7 @@ kind: PersistentVolumeClaim
 metadata:
   name: {{ project_name }}-artifacts-pv-claim
 spec:
-  storageClassName: manual
+  storageClassName: standard
   accessModes:
     - ReadWriteOnce
   resources:
@@ -15,7 +15,7 @@ kind: PersistentVolumeClaim
 metadata:
   name: {{ project_name }}-archives-pv-claim
 spec:
-  storageClassName: manual
+  storageClassName: standard
   accessModes:
     - ReadWriteOnce
   resources:
@@ -44,15 +44,15 @@ spec:
             - mountPath: {{ artifact_volume.mount_path }}
               name: artifacts
               subPath: {{ artifact_volume.sub_path }}
-	    {% for n in range(deps_volume | length) -%}
-	    - mountPath: {{ deps_volume[n].mount_path }}
-	      name: artifacts
-	      subPath: {{ deps_volume[n].sub_path }}
-	    {% endfor %}
-	    {%- for n in range(archives_volume | length) -%}
-	    - mountPath: {{ archives_volume[n].mount_path }}
-	      name: archives
-	      subPath: {{ archives_volume[n].sub_path }}
-	    {%- endfor %}
+            {% for n in range(deps_volume | length) -%}
+            - mountPath: {{ deps_volume[n].mount_path }}
+              name: artifacts
+              subPath: {{ deps_volume[n].sub_path }}
+            {% endfor %}
+            {%- for n in range(archives_volume | length) -%}
+            - mountPath: {{ archives_volume[n].mount_path }}
+              name: archives
+              subPath: {{ archives_volume[n].sub_path }}
+            {%- endfor %}
       restartPolicy: Never
   backoffLimit: 1
\ No newline at end of file

manifests-template/populate_template.py

@@ -11,6 +11,8 @@ parser.add_argument('step_name', metavar='STEP_NAME', type=str,
                     help='The name of the step to run')
 parser.add_argument('commit', metavar='COMMIT', type=str,
                     help='The commit of the src repo (tag of the corresponding image)')
+parser.add_argument('artifact_dir', metavar='ARTIFACT_DIR', type=str,
+                    help='The artifact step directory')
 parser.add_argument('deps_dir', metavar='DEPS_DIR', type=str, nargs='+',
                     help='The list of dependances directory')
 
@@ -30,8 +32,10 @@ step_config = config['steps'][args.step_name]
 # print(step_config)
 
 step_config['step_name'] = args.step_name
+step_config['commit'] = args.commit
 step_config['project_name'] = config['project_name']
 step_config['resources'] = config['resources']
+step_config['artifact_volume']['sub_path'] = args.artifact_dir
 
 for dep_dir in args.deps_dir:
 
@@ -43,7 +47,7 @@ for volume in step_config['archives_volume']:
     volume['sub_path'] = volume['name']
     
 
-print(step_config)
+#print(step_config)
 
 # WHERE steps.${step_name}.deps_volumes.name = DEPS_DIR[:: -27] (dep step_name)
 # steps.${step_name}.deps_volumes.sub_path = DEPS_DIR

mount_archives_to_minikube.sh

+#!/bin/env bash
+minikube mount archives-pv:/mnt/ml_plates_archives

mount_artifacts_to_minikube.sh

+#!/bin/env bash
+minikube mount artifacts-pv:/mnt/ml_plates_artifacts

src/run.sh

 function gen_dir_checksum() {
+    # The ON_PLACE param basically state is the folder name should be into the sha1 or not
     
     local DIR_PATH=$1
     local OUTPUT_FILENAME=$2
+    local ON_PLACE=$3 # make this an option (TODO)
 
     if [ -z ${DIR_PATH} ]; then echo "ERROR: you must provide the path to the directory to checksum"; return 2; fi;
 
@@ -18,9 +20,14 @@ function gen_dir_checksum() {
     local WD=`pwd`
     local DIR="${DIR_PATH##*/}"
 
-    cd $(realpath ${DIR_PATH}/..)
-
-    filenames=$(find "${DIR}" -type f) # find all files (hidden and backup)
+    if [ ! -z ${ON_PLACE} ];
+    then
+	cd $(realpath ${DIR_PATH})
+	filenames=$(find "." -type f) # find all files (hidden and backup)
+    else
+	cd $(realpath ${DIR_PATH}/..)
+	filenames=$(find "${DIR}" -type f) # find all files (hidden and backup)
+    fi;
 
     touch ${OUTPUT_FILENAME}
     

templates/default.yaml

+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ step_name }}
+spec:
+  template:
+    spec:
+      volumes:
+        - name: artifacts
+          persistentVolumeClaim:
+            claimName: {{ project_name }}-artifacts-pv-claim
+        - name: archives
+          persistentVolumeClaim:
+            claimName: {{ project_name }}-archives-pv-claim	    
+      containers:
+        - name: {{ step_name }}
+          image: {{ step_name }}:{{ commit }}
+          imagePullPolicy: Never
+          volumeMounts:
+            - mountPath: {{ artifact_volume.mount_path }}
+              name: artifacts
+              subPath: {{ artifact_volume.sub_path }}
+            {% for n in range(deps_volume | length) -%}
+            - mountPath: {{ deps_volume[n].mount_path }}
+              name: artifacts
+              subPath: {{ deps_volume[n].sub_path }}
+            {% endfor %}
+            {%- for n in range(archives_volume | length) -%}
+            - mountPath: {{ archives_volume[n].mount_path }}
+              name: archives
+              subPath: {{ archives_volume[n].sub_path }}
+            {%- endfor %}
+      restartPolicy: Never
+  backoffLimit: 1
\ No newline at end of file

templates/pv-claim.yaml

+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: ml-plate-artifacts
+spec:
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 10Gi
+  claimRef:
+    name: ml-solar-plates-artifacts-pv-claim
+  hostPath:
+    path: /mnt/ml_plates_artifacts
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: standard
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: ml-plate-archives
+spec:
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 10Gi
+  claimRef:
+    name: ml-solar-plates-archives-pv-claim
+  hostPath:
+    path: /mnt/ml_plates_archives
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: standard
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ml-solar-plates-artifacts-pv-claim
+spec:
+  storageClassName: standard
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ml-solar-plates-archives-pv-claim
+spec:
+  storageClassName: standard
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
\ No newline at end of file