Commit 37ecb795 authored by Yori Fournier's avatar Yori Fournier
Browse files

almost done with run_step

parent 35838748
......@@ -24,6 +24,7 @@
10. setup env
PWD=`pwd`
export PATH=$PATH:${PWD}/bin
export PATH=$PATH:${PWD}/manifests-template/bin
11. setup-pipeline.sh (uses config.toml)
......
apiVersion: v1
kind: PersistentVolume
metadata:
name: ml-plate-archives
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 10Gi
claimRef:
name: ml-solar-plates-archives-pv-claim
hostPath:
path: /mnt/ml_plates_archives
persistentVolumeReclaimPolicy: Retain
storageClassName: standard
apiVersion: v1
kind: PersistentVolume
metadata:
name: ml-plate-artifacts
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 10Gi
claimRef:
name: ml-solar-plates-artifacts-pv-claim
hostPath:
path: /mnt/ml_plates_artifacts
persistentVolumeReclaimPolicy: Retain
storageClassName: standard
......@@ -22,6 +22,19 @@ if [[ ! " ${STEPS[*]} " =~ " ${STEP_NAME} " ]]; then
exit 1
fi
# Check dependancies
if [ ! -z ${DEPS_DIR} ]
then
for dep_dir in ${DEPS_DIR[@]};
do
if [ ! -d ${PIPE_ROOT}/${ARTIFACTS}/${dep_dir} ]
then
echo "Error dependance does not exists!"
exit
fi
done
fi
# setup tmp folder
TMP_HASH=$(gen_hash_word)
TMP_HASH=$(get_hash_from_word ${TMP_HASH})
......@@ -107,9 +120,45 @@ fi;
# run the step if it does not exist
ARTIFACT_HASH=$(gen_hash_word --seed ${STEP_DIR_NAME})
# ARTIFACT_HASH=$(gen_hash_word)
echo "artifact_hash = \"${ARTIFACT_HASH}\"" >> ${TMP_DIR}/trace.toml
# generate the step manifest
if [ -z ${DEPS_DIR} ];
then
gen-manifest.py ${STEP_NAME} ${COMMIT} .${STEP_NAME}_${TMP_HASH} --template templates/default.yaml --config config.toml --output ${TMP_DIR}/manifest.yaml
else
gen-manifest.py ${STEP_NAME} ${COMMIT} .${STEP_NAME}_${TMP_HASH} --deps-dir ${DEPS_DIR[@]} --template templates/default.yaml --config config.toml --output ${TMP_DIR}/manifest.yaml
fi;
if [[ $? != 0 ]]; then
echo "Could not generate the manifest. Please check the template."
exit 1
fi;
# make sure the files are accessible via the docker group
chown -R fournier:docker artifacts-pv
chown -R fournier:docker archives-pv
# apply the step manifest
kubectl apply -f ${TMP_DIR}/manifest.yaml
# waiting
echo
echo -n "WAITING for Job ${STEP_NAME} to finish..."
sleep 2;
until [ -z `kubectl get jobs ${STEP_NAME} -o jsonpath='{.status.active}'` ] ; do sleep 1; echo -n "." ; done
status=$(kubectl get jobs ${STEP_NAME} -o=jsonpath='{.status.conditions[].type}')
echo
echo "Job ${STEP_NAME} finished with status: ${status}"
echo
if [[ "${status}" != "Complete" ]]; then
echo "Job ${STEP_NAME} has failed. Job was not deleted."
echo "To remove it, please use:"
echo "kubectl delete -f ${TMP_DIR}/manifest.yaml"
exit 1
fi;
# add the dependancies to the trace
cat ${TMP_DIR}/trace_deps.toml >> ${TMP_DIR}/trace.toml
......@@ -122,6 +171,13 @@ cp -r ${PIPE_ROOT}/${RUNTIME_PARAMS}/${STEP_NAME} ${TMP_DIR}/runtime-parameters
mkdir ${TMP_DIR}/docker-image
cp ${PIPE_ROOT}/${IMAGES_ARCH}/${STEP_NAME}_${COMMIT}.tar ${TMP_DIR}/docker-image/
# gen the Artifact hash (incl. trace.toml)
gen_dir_checksum ${TMP_DIR} ${TMP_DIR}/artifact_checksum.sha1 "True"
if [[ $? != 0 ]]; then echo "Error in making checksum of the Artifacts"; exit 1; fi;
ARTIFACT_HASH=$(gen_hash_from_checksum ${TMP_DIR}/artifact_checksum.sha1)
echo ${ARTIFACT_HASH} > ${TMP_DIR}/hash.sha1
# rename the folder with the artifact short-hash
short_deps_hash=$(get_short_hash_from_word ${DEPS_HASH})
short_step_hash=$(get_short_hash_from_word ${STEP_HASH})
......@@ -132,4 +188,12 @@ mv ${TMP_DIR} ${PIPE_ROOT}/${ARTIFACTS}/${STEP_DIR_NAME}
echo "${STEP_DIR_NAME}"
read -p "Do you want to remove the Job? (y/n)" resp
if [[ "${resp}" == "y" ]]; then
kubectl delete -f ${PIPE_ROOT}/${ARTIFACTS}/${STEP_DIR_NAME}/manifest.yaml
else
echo "To remove it, please use:"
echo "kubectl delete -f ${PIPE_ROOT}/${ARTIFACTS}/${STEP_DIR_NAME}/manifest.yaml"
fi;
exit 0
......@@ -8,11 +8,11 @@ pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-
[resources.archives_volume]
storage = "10Gi"
storage = "2Gi"
[resources.artifacts_volume]
storage = "10Gi"
storage = "2Gi"
# STEPS: define the configuration of the steps: repo, commit, volumes mount point...
......@@ -35,7 +35,7 @@ pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-
# The list of archives and where they will be mounted in the container
[[steps.extract-features.archives_volume]]
name = "ml_solar_plates_mock"
mount_path = "/home/fournier/ml_solar_plates"
mount_path = "/home/fournier/solar_plates"
[steps.label-features]
......@@ -64,7 +64,7 @@ pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/select-meaningful-plates-and-features.git"
commit = "81ca31c"
commit = "1fb3e7c16dcd254b31a1edbd6bff542b3382ab5b"
# The archives to be mounted
archives = ["ml_solar_plates_mock"]
......@@ -79,7 +79,7 @@ pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-
# The list of archives and where they will be mounted in the container
[[steps.select-meaningful-plates-and-features.archives_volume]]
name = "ml_solar_plates_mock"
mount_path = "home/fournier/ml_solar_plates"
mount_path = "home/fournier/solar_plates"
# The list of dependancies and where they are mounted
[[steps.select-meaningful-plates-and-features.deps_volume]]
......
#!/bin/env python
from jinja2 import Template
import toml
# INPUT PARAM ARE:
# STEP_NAME, COMMTI, DEPS_DIR (alike for run_step.sh)
import argparse
parser = argparse.ArgumentParser(description='Populate a step manifest template with pipeline config')
parser.add_argument('step_name', metavar='STEP_NAME', type=str,
help='The name of the step to run')
parser.add_argument('commit', metavar='COMMIT', type=str,
help='The commit of the src repo (tag of the corresponding image)')
parser.add_argument('artifact_dir', metavar='ARTIFACT_DIR', type=str,
help='The artifact step directory')
parser.add_argument('--deps-dir', metavar='DEPS_DIR', type=str, nargs='+',
default=[], help='The list of dependances directory')
parser.add_argument('--template', metavar='TEMPLATE', type=str,
default=None, help='The template yaml file')
parser.add_argument('--config', metavar='CONFIG', type=str,
default=None, help='The config toml file')
parser.add_argument('--output', metavar='MANIFEST', type=str,
default=None, help='The output yaml file')
args = parser.parse_args()
if args.template is None:
template_filename = "templates/default.yaml"
else:
template_filename = args.template
if args.config is None:
config_filename = "config.toml"
else:
config_filename = args.config
with open(template_filename, 'r') as fd:
template = Template(fd.read())
with open(config_filename, 'r') as fd:
config = toml.load(fd)
# update config with
step_config = config['steps'][args.step_name]
step_config['step_name'] = args.step_name
step_config['commit'] = args.commit
step_config['project_name'] = config['project_name']
step_config['resources'] = config['resources']
step_config['artifact_volume']['sub_path'] = args.artifact_dir
for dep_dir in args.deps_dir:
for volume in step_config['deps_volume']:
if volume['name'] == dep_dir[0:-27]: # name of the step
volume['sub_path'] = dep_dir
for volume in step_config.get('archives_volume', []):
volume['sub_path'] = volume['name']
rendered = template.render(**step_config)
if args.output is not None:
with open(args.output, 'w') as fd:
fd.write(rendered)
else:
print(rendered)
# sources:
# https://stackoverflow.com/a/50557523
# https://stackoverflow.com/a/50333856
......@@ -3,7 +3,7 @@ kind: PersistentVolumeClaim
metadata:
name: {{ project_name }}-artifacts-pv-claim
spec:
storageClassName: manual
storageClassName: standard
accessModes:
- ReadWriteOnce
resources:
......@@ -15,7 +15,7 @@ kind: PersistentVolumeClaim
metadata:
name: {{ project_name }}-archives-pv-claim
spec:
storageClassName: manual
storageClassName: standard
accessModes:
- ReadWriteOnce
resources:
......@@ -44,15 +44,15 @@ spec:
- mountPath: {{ artifact_volume.mount_path }}
name: artifacts
subPath: {{ artifact_volume.sub_path }}
{% for n in range(deps_volume | length) -%}
- mountPath: {{ deps_volume[n].mount_path }}
name: artifacts
subPath: {{ deps_volume[n].sub_path }}
{% endfor %}
{%- for n in range(archives_volume | length) -%}
- mountPath: {{ archives_volume[n].mount_path }}
name: archives
subPath: {{ archives_volume[n].sub_path }}
{%- endfor %}
{% for n in range(deps_volume | length) -%}
- mountPath: {{ deps_volume[n].mount_path }}
name: artifacts
subPath: {{ deps_volume[n].sub_path }}
{% endfor %}
{%- for n in range(archives_volume | length) -%}
- mountPath: {{ archives_volume[n].mount_path }}
name: archives
subPath: {{ archives_volume[n].sub_path }}
{%- endfor %}
restartPolicy: Never
backoffLimit: 1
\ No newline at end of file
......@@ -11,6 +11,8 @@ parser.add_argument('step_name', metavar='STEP_NAME', type=str,
help='The name of the step to run')
parser.add_argument('commit', metavar='COMMIT', type=str,
help='The commit of the src repo (tag of the corresponding image)')
parser.add_argument('artifact_dir', metavar='ARTIFACT_DIR', type=str,
help='The artifact step directory')
parser.add_argument('deps_dir', metavar='DEPS_DIR', type=str, nargs='+',
help='The list of dependances directory')
......@@ -30,8 +32,10 @@ step_config = config['steps'][args.step_name]
# print(step_config)
step_config['step_name'] = args.step_name
step_config['commit'] = args.commit
step_config['project_name'] = config['project_name']
step_config['resources'] = config['resources']
step_config['artifact_volume']['sub_path'] = args.artifact_dir
for dep_dir in args.deps_dir:
......@@ -43,7 +47,7 @@ for volume in step_config['archives_volume']:
volume['sub_path'] = volume['name']
print(step_config)
#print(step_config)
# WHERE steps.${step_name}.deps_volumes.name = DEPS_DIR[:: -27] (dep step_name)
# steps.${step_name}.deps_volumes.sub_path = DEPS_DIR
......
#!/bin/env bash
minikube mount archives-pv:/mnt/ml_plates_archives
#!/bin/env bash
minikube mount artifacts-pv:/mnt/ml_plates_artifacts
function gen_dir_checksum() {
# The ON_PLACE param basically state is the folder name should be into the sha1 or not
local DIR_PATH=$1
local OUTPUT_FILENAME=$2
local ON_PLACE=$3 # make this an option (TODO)
if [ -z ${DIR_PATH} ]; then echo "ERROR: you must provide the path to the directory to checksum"; return 2; fi;
......@@ -18,9 +20,14 @@ function gen_dir_checksum() {
local WD=`pwd`
local DIR="${DIR_PATH##*/}"
cd $(realpath ${DIR_PATH}/..)
filenames=$(find "${DIR}" -type f) # find all files (hidden and backup)
if [ ! -z ${ON_PLACE} ];
then
cd $(realpath ${DIR_PATH})
filenames=$(find "." -type f) # find all files (hidden and backup)
else
cd $(realpath ${DIR_PATH}/..)
filenames=$(find "${DIR}" -type f) # find all files (hidden and backup)
fi;
touch ${OUTPUT_FILENAME}
......
apiVersion: batch/v1
kind: Job
metadata:
name: {{ step_name }}
spec:
template:
spec:
volumes:
- name: artifacts
persistentVolumeClaim:
claimName: {{ project_name }}-artifacts-pv-claim
- name: archives
persistentVolumeClaim:
claimName: {{ project_name }}-archives-pv-claim
containers:
- name: {{ step_name }}
image: {{ step_name }}:{{ commit }}
imagePullPolicy: Never
volumeMounts:
- mountPath: {{ artifact_volume.mount_path }}
name: artifacts
subPath: {{ artifact_volume.sub_path }}
{% for n in range(deps_volume | length) -%}
- mountPath: {{ deps_volume[n].mount_path }}
name: artifacts
subPath: {{ deps_volume[n].sub_path }}
{% endfor %}
{%- for n in range(archives_volume | length) -%}
- mountPath: {{ archives_volume[n].mount_path }}
name: archives
subPath: {{ archives_volume[n].sub_path }}
{%- endfor %}
restartPolicy: Never
backoffLimit: 1
\ No newline at end of file
apiVersion: v1
kind: PersistentVolume
metadata:
name: ml-plate-artifacts
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 10Gi
claimRef:
name: ml-solar-plates-artifacts-pv-claim
hostPath:
path: /mnt/ml_plates_artifacts
persistentVolumeReclaimPolicy: Retain
storageClassName: standard
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: ml-plate-archives
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 10Gi
claimRef:
name: ml-solar-plates-archives-pv-claim
hostPath:
path: /mnt/ml_plates_archives
persistentVolumeReclaimPolicy: Retain
storageClassName: standard
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ml-solar-plates-artifacts-pv-claim
spec:
storageClassName: standard
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ml-solar-plates-archives-pv-claim
spec:
storageClassName: standard
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment