apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: generateName: object-detection-example- annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.4.0, pipelines.kubeflow.org/pipeline_compilation_time: '2022-09-01T16:01:39.268852', pipelines.kubeflow.org/pipeline_spec: '{"description": "Continues training a pretrained pet detection model, then tests serving it.", "inputs": [{"default": "http://www.robots.ox.ac.uk/~vgg/data/pets/data/images.tar.gz", "name": "images", "optional": true}, {"default": "http://www.robots.ox.ac.uk/~vgg/data/pets/data/annotations.tar.gz", "name": "annotations", "optional": true}, {"default": "http://storage.googleapis.com/download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_coco_11_06_2017.tar.gz", "name": "pretrained", "optional": true}], "name": "Object Detection Example"}'} labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.4.0} spec: entrypoint: object-detection-example templates: - name: load-task container: args: [--images, '{{inputs.parameters.images}}', --annotations, '{{inputs.parameters.annotations}}', --records, /tmp/outputs/records/data, --validation-images, /tmp/outputs/validation_images/data] command: - sh - -ec - | program_path=$(mktemp) printf "%s" "$0" > "$program_path" python3 -u "$program_path" "$@" - | def _parent_dirs_maker_that_returns_open_file(mode: str, encoding: str = None): def make_parent_dirs_and_return_path(file_path: str): import os os.makedirs(os.path.dirname(file_path), exist_ok=True) return open(file_path, mode=mode, encoding=encoding) return make_parent_dirs_and_return_path def load_task( images, annotations, records, validation_images, ): """Transforms pet data from images to TensorFlow records.""" from glob import glob from pathlib import Path from tensorflow.python.keras.utils import get_file import subprocess import tarfile def load(path): return get_file(Path(path).name, path, extract=True) load(images) load(annotations) with tarfile.open(mode='w:gz', fileobj=validation_images) as tar: for image in glob('/root/.keras/datasets/images/*.jpg')[:10]: tar.add(image, arcname=Path(image).name) subprocess.run( [ 'python', 'object_detection/dataset_tools/create_pet_tf_record.py', '--label_map_path=object_detection/data/pet_label_map.pbtxt', '--data_dir', '/root/.keras/datasets/', '--output_dir=/models/research', ], check=True, cwd='/models/research', ) with tarfile.open(mode='w:gz', fileobj=records) as tar: for record in glob('/models/research/*.record-*'): tar.add(record, arcname=Path(record).name) import argparse _parser = argparse.ArgumentParser(prog='Load task', description='Transforms pet data from images to TensorFlow records.') _parser.add_argument("--images", dest="images", type=str, required=True, default=argparse.SUPPRESS) _parser.add_argument("--annotations", dest="annotations", type=str, required=True, default=argparse.SUPPRESS) _parser.add_argument("--records", dest="records", type=_parent_dirs_maker_that_returns_open_file('wb'), required=True, default=argparse.SUPPRESS) _parser.add_argument("--validation-images", dest="validation_images", type=_parent_dirs_maker_that_returns_open_file('wb'), required=True, default=argparse.SUPPRESS) _parsed_args = vars(_parser.parse_args()) _outputs = load_task(**_parsed_args) image: rocks.canonical.com:5000/kubeflow/examples/object_detection:latest resources: limits: {nvidia.com/gpu: 1} volumeMounts: - {mountPath: /output, name: volume} - {mountPath: /tmp/outputs, name: outputs} inputs: parameters: - {name: annotations} - {name: images} outputs: artifacts: - {name: mlpipeline-ui-metadata, path: /tmp/outputs/mlpipeline-ui-metadata.json} - {name: mlpipeline-metrics, path: /tmp/outputs/mlpipeline-metrics.json} - {name: load-task-records, path: /tmp/outputs/records/data} - {name: load-task-validation_images, path: /tmp/outputs/validation_images/data} volumes: - emptyDir: {} name: outputs - emptyDir: {} name: volume metadata: annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Transforms pet data from images to TensorFlow records.", "implementation": {"container": {"args": ["--images", {"inputValue": "images"}, "--annotations", {"inputValue": "annotations"}, "--records", {"outputPath": "records"}, "--validation-images", {"outputPath": "validation_images"}], "command": ["sh", "-ec", "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", "def _parent_dirs_maker_that_returns_open_file(mode: str, encoding: str = None):\n def make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return open(file_path, mode=mode, encoding=encoding)\n return make_parent_dirs_and_return_path\n\ndef load_task(\n images,\n annotations,\n records,\n validation_images,\n):\n \"\"\"Transforms pet data from images to TensorFlow records.\"\"\"\n\n from glob import glob\n from pathlib import Path\n from tensorflow.python.keras.utils import get_file\n import subprocess\n import tarfile\n\n def load(path):\n return get_file(Path(path).name, path, extract=True)\n\n load(images)\n load(annotations)\n\n with tarfile.open(mode=''w:gz'', fileobj=validation_images) as tar:\n for image in glob(''/root/.keras/datasets/images/*.jpg'')[:10]:\n tar.add(image, arcname=Path(image).name)\n\n subprocess.run(\n [\n ''python'',\n ''object_detection/dataset_tools/create_pet_tf_record.py'',\n ''--label_map_path=object_detection/data/pet_label_map.pbtxt'',\n ''--data_dir'',\n ''/root/.keras/datasets/'',\n ''--output_dir=/models/research'',\n ],\n check=True,\n cwd=''/models/research'',\n )\n\n with tarfile.open(mode=''w:gz'', fileobj=records) as tar:\n for record in glob(''/models/research/*.record-*''):\n tar.add(record, arcname=Path(record).name)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog=''Load task'', description=''Transforms pet data from images to TensorFlow records.'')\n_parser.add_argument(\"--images\", dest=\"images\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--annotations\", dest=\"annotations\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--records\", dest=\"records\", type=_parent_dirs_maker_that_returns_open_file(''wb''), required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--validation-images\", dest=\"validation_images\", type=_parent_dirs_maker_that_returns_open_file(''wb''), required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = load_task(**_parsed_args)\n"], "image": "rocks.canonical.com:5000/kubeflow/examples/object_detection:latest"}}, "inputs": [{"name": "images", "type": "String"}, {"name": "annotations", "type": "String"}], "name": "Load task", "outputs": [{"name": "records", "type": "String"}, {"name": "validation_images", "type": "String"}]}', pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"annotations": "{{inputs.parameters.annotations}}", "images": "{{inputs.parameters.images}}"}'} - name: object-detection-example inputs: parameters: - {name: annotations} - {name: images} - {name: pretrained} dag: tasks: - name: load-task template: load-task arguments: parameters: - {name: annotations, value: '{{inputs.parameters.annotations}}'} - {name: images, value: '{{inputs.parameters.images}}'} - name: test-task template: test-task dependencies: [load-task, train-task] arguments: artifacts: - {name: load-task-validation_images, from: '{{tasks.load-task.outputs.artifacts.load-task-validation_images}}'} - {name: train-task-exported, from: '{{tasks.train-task.outputs.artifacts.train-task-exported}}'} - name: train-task template: train-task dependencies: [load-task] arguments: parameters: - {name: pretrained, value: '{{inputs.parameters.pretrained}}'} artifacts: - {name: load-task-records, from: '{{tasks.load-task.outputs.artifacts.load-task-records}}'} - name: test-task container: args: [--model, /tmp/inputs/model/data, --validation-images, /tmp/inputs/validation_images/data] command: - sh - -ec - | program_path=$(mktemp) printf "%s" "$0" > "$program_path" python3 -u "$program_path" "$@" - | def test_task(model, validation_images): """Connects to served model and tests example pet images.""" import numpy as np import requests import shutil import tarfile import time from matplotlib.pyplot import imread with tarfile.open(model.name) as tar: tar.extractall(path="/") shutil.move('/exported', '/output/object_detection') # https://stackoverflow.com/a/45552938 shutil.copytree('/output/object_detection/saved_model', '/output/object_detection/1') with tarfile.open(validation_images.name) as tar: tar.extractall(path="/images") model_url = 'http://localhost:9001/v1/models/object_detection' for _ in range(60): try: requests.get(f'{model_url}/versions/1').raise_for_status() break except requests.RequestException as err: print(err) time.sleep(5) else: raise Exception("Waited too long for sidecar to come up!") response = requests.get(f'{model_url}/metadata') response.raise_for_status() assert response.json() == { 'model_spec': {'name': 'object_detection', 'signature_name': '', 'version': '1'}, 'metadata': { 'signature_def': { 'signature_def': { 'serving_default': { 'inputs': { 'inputs': { 'dtype': 'DT_UINT8', 'tensor_shape': { 'dim': [ {'size': '-1', 'name': ''}, {'size': '-1', 'name': ''}, {'size': '-1', 'name': ''}, {'size': '3', 'name': ''}, ], 'unknown_rank': False, }, 'name': 'image_tensor:0', } }, 'outputs': { 'raw_detection_scores': { 'dtype': 'DT_FLOAT', 'tensor_shape': { 'dim': [ {'size': '-1', 'name': ''}, {'size': '300', 'name': ''}, {'size': '38', 'name': ''}, ], 'unknown_rank': False, }, 'name': 'raw_detection_scores:0', }, 'detection_multiclass_scores': { 'dtype': 'DT_FLOAT', 'tensor_shape': { 'dim': [ {'size': '-1', 'name': ''}, {'size': '300', 'name': ''}, {'size': '38', 'name': ''}, ], 'unknown_rank': False, }, 'name': 'detection_multiclass_scores:0', }, 'detection_classes': { 'dtype': 'DT_FLOAT', 'tensor_shape': { 'dim': [ {'size': '-1', 'name': ''}, {'size': '300', 'name': ''}, ], 'unknown_rank': False, }, 'name': 'detection_classes:0', }, 'num_detections': { 'dtype': 'DT_FLOAT', 'tensor_shape': { 'dim': [{'size': '-1', 'name': ''}], 'unknown_rank': False, }, 'name': 'num_detections:0', }, 'detection_boxes': { 'dtype': 'DT_FLOAT', 'tensor_shape': { 'dim': [ {'size': '-1', 'name': ''}, {'size': '300', 'name': ''}, {'size': '4', 'name': ''}, ], 'unknown_rank': False, }, 'name': 'detection_boxes:0', }, 'raw_detection_boxes': { 'dtype': 'DT_FLOAT', 'tensor_shape': { 'dim': [ {'size': '-1', 'name': ''}, {'size': '300', 'name': ''}, {'size': '4', 'name': ''}, ], 'unknown_rank': False, }, 'name': 'raw_detection_boxes:0', }, 'detection_scores': { 'dtype': 'DT_FLOAT', 'tensor_shape': { 'dim': [ {'size': '-1', 'name': ''}, {'size': '300', 'name': ''}, ], 'unknown_rank': False, }, 'name': 'detection_scores:0', }, }, 'method_name': 'tensorflow/serving/predict', } } } }, } test_images = np.zeros((5, 100, 100, 3), dtype=np.uint8).tolist() response = requests.post(f'{model_url}:predict', json={'instances': test_images}) response.raise_for_status() shapes = { 'detection_boxes': (300, 4), 'raw_detection_boxes': (300, 4), 'detection_scores': (300,), 'raw_detection_scores': (300, 38), 'detection_multiclass_scores': (300, 38), 'detection_classes': (300,), } for i, prediction in enumerate(response.json()['predictions']): print("Checking prediction #%s" % i) for name, shape in shapes.items(): assert np.array(prediction[name]).shape == shape, name with open('pet.jpg', 'wb') as f: f.write( requests.get( 'https://github.com/canonical/bundle-kubeflow/raw/main/tests/pipelines/artifacts/pet.jpg' ).content ) test_image = imread('pet.jpg').reshape((1, 500, 357, 3)).tolist() response = requests.post(f'{model_url}:predict', json={'instances': test_image}) response.raise_for_status() shapes = { 'detection_boxes': (300, 4), 'raw_detection_boxes': (300, 4), 'detection_scores': (300,), 'raw_detection_scores': (300, 38), 'detection_multiclass_scores': (300, 38), 'detection_classes': (300,), } for i, prediction in enumerate(response.json()['predictions']): print("Checking prediction #%s" % i) for name, shape in shapes.items(): assert np.array(prediction[name]).shape == shape, name import argparse _parser = argparse.ArgumentParser(prog='Test task', description='Connects to served model and tests example pet images.') _parser.add_argument("--model", dest="model", type=argparse.FileType('rb'), required=True, default=argparse.SUPPRESS) _parser.add_argument("--validation-images", dest="validation_images", type=argparse.FileType('rb'), required=True, default=argparse.SUPPRESS) _parsed_args = vars(_parser.parse_args()) _outputs = test_task(**_parsed_args) image: rocks.canonical.com:5000/kubeflow/examples/object_detection:latest volumeMounts: - {mountPath: /output, name: volume} - {mountPath: /tmp/outputs, name: outputs} inputs: artifacts: - {name: train-task-exported, path: /tmp/inputs/model/data} - {name: load-task-validation_images, path: /tmp/inputs/validation_images/data} outputs: artifacts: - {name: mlpipeline-ui-metadata, path: /tmp/outputs/mlpipeline-ui-metadata.json} - {name: mlpipeline-metrics, path: /tmp/outputs/mlpipeline-metrics.json} sidecars: - args: [--model_name=object_detection, --model_base_path=/output/object_detection, --port=9000, --rest_api_port=9001] command: [/usr/bin/tensorflow_model_server] image: tensorflow/serving:1.15.0-gpu name: tensorflow-serve resources: limits: {nvidia.com/gpu: 1} mirrorVolumeMounts: true volumes: - emptyDir: {} name: outputs - emptyDir: {} name: volume metadata: annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Connects to served model and tests example pet images.", "implementation": {"container": {"args": ["--model", {"inputPath": "model"}, "--validation-images", {"inputPath": "validation_images"}], "command": ["sh", "-ec", "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", "def test_task(model, validation_images):\n \"\"\"Connects to served model and tests example pet images.\"\"\"\n\n import numpy as np\n import requests\n import shutil\n import tarfile\n import time\n from matplotlib.pyplot import imread\n\n with tarfile.open(model.name) as tar:\n tar.extractall(path=\"/\")\n shutil.move(''/exported'', ''/output/object_detection'')\n # https://stackoverflow.com/a/45552938\n shutil.copytree(''/output/object_detection/saved_model'', ''/output/object_detection/1'')\n\n with tarfile.open(validation_images.name) as tar:\n tar.extractall(path=\"/images\")\n\n model_url = ''http://localhost:9001/v1/models/object_detection''\n for _ in range(60):\n try:\n requests.get(f''{model_url}/versions/1'').raise_for_status()\n break\n except requests.RequestException as err:\n print(err)\n time.sleep(5)\n else:\n raise Exception(\"Waited too long for sidecar to come up!\")\n\n response = requests.get(f''{model_url}/metadata'')\n response.raise_for_status()\n assert response.json() == {\n ''model_spec'': {''name'': ''object_detection'', ''signature_name'': '''', ''version'': ''1''},\n ''metadata'': {\n ''signature_def'': {\n ''signature_def'': {\n ''serving_default'': {\n ''inputs'': {\n ''inputs'': {\n ''dtype'': ''DT_UINT8'',\n ''tensor_shape'': {\n ''dim'': [\n {''size'': ''-1'', ''name'': ''''},\n {''size'': ''-1'', ''name'': ''''},\n {''size'': ''-1'', ''name'': ''''},\n {''size'': ''3'', ''name'': ''''},\n ],\n ''unknown_rank'': False,\n },\n ''name'': ''image_tensor:0'',\n }\n },\n ''outputs'': {\n ''raw_detection_scores'': {\n ''dtype'': ''DT_FLOAT'',\n ''tensor_shape'': {\n ''dim'': [\n {''size'': ''-1'', ''name'': ''''},\n {''size'': ''300'', ''name'': ''''},\n {''size'': ''38'', ''name'': ''''},\n ],\n ''unknown_rank'': False,\n },\n ''name'': ''raw_detection_scores:0'',\n },\n ''detection_multiclass_scores'': {\n ''dtype'': ''DT_FLOAT'',\n ''tensor_shape'': {\n ''dim'': [\n {''size'': ''-1'', ''name'': ''''},\n {''size'': ''300'', ''name'': ''''},\n {''size'': ''38'', ''name'': ''''},\n ],\n ''unknown_rank'': False,\n },\n ''name'': ''detection_multiclass_scores:0'',\n },\n ''detection_classes'': {\n ''dtype'': ''DT_FLOAT'',\n ''tensor_shape'': {\n ''dim'': [\n {''size'': ''-1'', ''name'': ''''},\n {''size'': ''300'', ''name'': ''''},\n ],\n ''unknown_rank'': False,\n },\n ''name'': ''detection_classes:0'',\n },\n ''num_detections'': {\n ''dtype'': ''DT_FLOAT'',\n ''tensor_shape'': {\n ''dim'': [{''size'': ''-1'', ''name'': ''''}],\n ''unknown_rank'': False,\n },\n ''name'': ''num_detections:0'',\n },\n ''detection_boxes'': {\n ''dtype'': ''DT_FLOAT'',\n ''tensor_shape'': {\n ''dim'': [\n {''size'': ''-1'', ''name'': ''''},\n {''size'': ''300'', ''name'': ''''},\n {''size'': ''4'', ''name'': ''''},\n ],\n ''unknown_rank'': False,\n },\n ''name'': ''detection_boxes:0'',\n },\n ''raw_detection_boxes'': {\n ''dtype'': ''DT_FLOAT'',\n ''tensor_shape'': {\n ''dim'': [\n {''size'': ''-1'', ''name'': ''''},\n {''size'': ''300'', ''name'': ''''},\n {''size'': ''4'', ''name'': ''''},\n ],\n ''unknown_rank'': False,\n },\n ''name'': ''raw_detection_boxes:0'',\n },\n ''detection_scores'': {\n ''dtype'': ''DT_FLOAT'',\n ''tensor_shape'': {\n ''dim'': [\n {''size'': ''-1'', ''name'': ''''},\n {''size'': ''300'', ''name'': ''''},\n ],\n ''unknown_rank'': False,\n },\n ''name'': ''detection_scores:0'',\n },\n },\n ''method_name'': ''tensorflow/serving/predict'',\n }\n }\n }\n },\n }\n\n test_images = np.zeros((5, 100, 100, 3), dtype=np.uint8).tolist()\n response = requests.post(f''{model_url}:predict'', json={''instances'': test_images})\n response.raise_for_status()\n shapes = {\n ''detection_boxes'': (300, 4),\n ''raw_detection_boxes'': (300, 4),\n ''detection_scores'': (300,),\n ''raw_detection_scores'': (300, 38),\n ''detection_multiclass_scores'': (300, 38),\n ''detection_classes'': (300,),\n }\n for i, prediction in enumerate(response.json()[''predictions'']):\n print(\"Checking prediction #%s\" % i)\n for name, shape in shapes.items():\n assert np.array(prediction[name]).shape == shape, name\n\n with open(''pet.jpg'', ''wb'') as f:\n f.write(\n requests.get(\n ''https://github.com/canonical/bundle-kubeflow/raw/main/tests/pipelines/artifacts/pet.jpg''\n ).content\n )\n test_image = imread(''pet.jpg'').reshape((1, 500, 357, 3)).tolist()\n response = requests.post(f''{model_url}:predict'', json={''instances'': test_image})\n response.raise_for_status()\n shapes = {\n ''detection_boxes'': (300, 4),\n ''raw_detection_boxes'': (300, 4),\n ''detection_scores'': (300,),\n ''raw_detection_scores'': (300, 38),\n ''detection_multiclass_scores'': (300, 38),\n ''detection_classes'': (300,),\n }\n for i, prediction in enumerate(response.json()[''predictions'']):\n print(\"Checking prediction #%s\" % i)\n for name, shape in shapes.items():\n assert np.array(prediction[name]).shape == shape, name\n\nimport argparse\n_parser = argparse.ArgumentParser(prog=''Test task'', description=''Connects to served model and tests example pet images.'')\n_parser.add_argument(\"--model\", dest=\"model\", type=argparse.FileType(''rb''), required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--validation-images\", dest=\"validation_images\", type=argparse.FileType(''rb''), required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = test_task(**_parsed_args)\n"], "image": "rocks.canonical.com:5000/kubeflow/examples/object_detection:latest"}}, "inputs": [{"name": "model", "type": "String"}, {"name": "validation_images", "type": "String"}], "name": "Test task"}', pipelines.kubeflow.org/component_ref: '{}'} - name: train-task container: args: [--records, /tmp/inputs/records/data, --pretrained, '{{inputs.parameters.pretrained}}', --exported, /tmp/outputs/exported/data] command: - sh - -ec - | program_path=$(mktemp) printf "%s" "$0" > "$program_path" python3 -u "$program_path" "$@" - | def _parent_dirs_maker_that_returns_open_file(mode: str, encoding: str = None): def make_parent_dirs_and_return_path(file_path: str): import os os.makedirs(os.path.dirname(file_path), exist_ok=True) return open(file_path, mode=mode, encoding=encoding) return make_parent_dirs_and_return_path def train_task(records, pretrained, exported): from pathlib import Path from tensorflow.python.keras.utils import get_file import subprocess import shutil import re import tarfile import sys def load(path): return get_file(Path(path).name, path, extract=True) model_path = Path(load(pretrained)) model_path = str(model_path.with_name(model_path.name.split('.')[0])) shutil.move(model_path, '/model') with tarfile.open(mode='r:gz', fileobj=records) as tar: tar.extractall('/records') with open('/pipeline.config', 'w') as f: config = Path('samples/configs/faster_rcnn_resnet101_pets.config').read_text() config = re.sub(r'PATH_TO_BE_CONFIGURED\/model\.ckpt', '/model/model.ckpt', config) config = re.sub('PATH_TO_BE_CONFIGURED', '/records', config) f.write(config) shutil.copy('data/pet_label_map.pbtxt', '/records/pet_label_map.pbtxt') print("Training model") subprocess.check_call( [ sys.executable, 'model_main.py', '--model_dir', '/model', '--num_train_steps', '1', '--pipeline_config_path', '/pipeline.config', ], ) subprocess.check_call( [ sys.executable, 'export_inference_graph.py', '--input_type', 'image_tensor', '--pipeline_config_path', '/pipeline.config', '--trained_checkpoint_prefix', '/model/model.ckpt-1', '--output_directory', '/exported', ], ) with tarfile.open(mode='w:gz', fileobj=exported) as tar: tar.add('/exported', recursive=True) import argparse _parser = argparse.ArgumentParser(prog='Train task', description='') _parser.add_argument("--records", dest="records", type=argparse.FileType('rb'), required=True, default=argparse.SUPPRESS) _parser.add_argument("--pretrained", dest="pretrained", type=str, required=True, default=argparse.SUPPRESS) _parser.add_argument("--exported", dest="exported", type=_parent_dirs_maker_that_returns_open_file('wb'), required=True, default=argparse.SUPPRESS) _parsed_args = vars(_parser.parse_args()) _outputs = train_task(**_parsed_args) image: rocks.canonical.com:5000/kubeflow/examples/object_detection:latest resources: limits: {nvidia.com/gpu: 1} volumeMounts: - {mountPath: /output, name: volume} - {mountPath: /tmp/outputs, name: outputs} inputs: parameters: - {name: pretrained} artifacts: - {name: load-task-records, path: /tmp/inputs/records/data} outputs: artifacts: - {name: mlpipeline-ui-metadata, path: /tmp/outputs/mlpipeline-ui-metadata.json} - {name: mlpipeline-metrics, path: /tmp/outputs/mlpipeline-metrics.json} - {name: train-task-exported, path: /tmp/outputs/exported/data} volumes: - emptyDir: {} name: outputs - emptyDir: {} name: volume metadata: annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": {"args": ["--records", {"inputPath": "records"}, "--pretrained", {"inputValue": "pretrained"}, "--exported", {"outputPath": "exported"}], "command": ["sh", "-ec", "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", "def _parent_dirs_maker_that_returns_open_file(mode: str, encoding: str = None):\n def make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return open(file_path, mode=mode, encoding=encoding)\n return make_parent_dirs_and_return_path\n\ndef train_task(records, pretrained, exported):\n from pathlib import Path\n from tensorflow.python.keras.utils import get_file\n import subprocess\n import shutil\n import re\n import tarfile\n import sys\n\n def load(path):\n return get_file(Path(path).name, path, extract=True)\n\n model_path = Path(load(pretrained))\n model_path = str(model_path.with_name(model_path.name.split(''.'')[0]))\n shutil.move(model_path, ''/model'')\n\n with tarfile.open(mode=''r:gz'', fileobj=records) as tar:\n tar.extractall(''/records'')\n\n with open(''/pipeline.config'', ''w'') as f:\n config = Path(''samples/configs/faster_rcnn_resnet101_pets.config'').read_text()\n config = re.sub(r''PATH_TO_BE_CONFIGURED\\/model\\.ckpt'', ''/model/model.ckpt'', config)\n config = re.sub(''PATH_TO_BE_CONFIGURED'', ''/records'', config)\n f.write(config)\n\n shutil.copy(''data/pet_label_map.pbtxt'', ''/records/pet_label_map.pbtxt'')\n\n print(\"Training model\")\n subprocess.check_call(\n [\n sys.executable,\n ''model_main.py'',\n ''--model_dir'',\n ''/model'',\n ''--num_train_steps'',\n ''1'',\n ''--pipeline_config_path'',\n ''/pipeline.config'',\n ],\n )\n\n subprocess.check_call(\n [\n sys.executable,\n ''export_inference_graph.py'',\n ''--input_type'',\n ''image_tensor'',\n ''--pipeline_config_path'',\n ''/pipeline.config'',\n ''--trained_checkpoint_prefix'',\n ''/model/model.ckpt-1'',\n ''--output_directory'',\n ''/exported'',\n ],\n )\n\n with tarfile.open(mode=''w:gz'', fileobj=exported) as tar:\n tar.add(''/exported'', recursive=True)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog=''Train task'', description='''')\n_parser.add_argument(\"--records\", dest=\"records\", type=argparse.FileType(''rb''), required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--pretrained\", dest=\"pretrained\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--exported\", dest=\"exported\", type=_parent_dirs_maker_that_returns_open_file(''wb''), required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = train_task(**_parsed_args)\n"], "image": "rocks.canonical.com:5000/kubeflow/examples/object_detection:latest"}}, "inputs": [{"name": "records", "type": "String"}, {"name": "pretrained", "type": "String"}], "name": "Train task", "outputs": [{"name": "exported", "type": "String"}]}', pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"pretrained": "{{inputs.parameters.pretrained}}"}'} arguments: parameters: - {name: images, value: 'http://www.robots.ox.ac.uk/~vgg/data/pets/data/images.tar.gz'} - {name: annotations, value: 'http://www.robots.ox.ac.uk/~vgg/data/pets/data/annotations.tar.gz'} - {name: pretrained, value: 'http://storage.googleapis.com/download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_coco_11_06_2017.tar.gz'} serviceAccountName: pipeline-runner