Live Model Server Testing#
Test model server via HTTP calls
# nuclio: ignore
import nuclio
%nuclio config spec.image = "mlrun/mlrun"
%nuclio: setting spec.image to 'mlrun/mlrun'
import os
import pandas as pd
import requests
import json
import numpy as np
from datetime import datetime
from mlrun.datastore import DataItem
from mlrun.artifacts import get_model, ChartArtifact
def model_server_tester(context,
table: DataItem,
addr: str,
label_column: str = "label",
model: str = '',
match_err: bool = False,
rows: int = 20):
""" Test a model server
:param table: csv/parquet table with test data
:param addr: function address/url
:param label_column: name of the label column in table
:param model: tested model name
:param match_err: raise error on validation (require proper test set)
:param rows: number of rows to use from test set
"""
table = table.as_df()
y_list = table.pop(label_column).values.tolist()
context.logger.info(f'testing with dataset against {addr}, model: {model}')
if rows and rows < table.shape[0]:
table = table.sample(rows)
count = err_count = match = 0
times = []
for x, y in zip(table.values, y_list):
count += 1
event_data = json.dumps({"instances":[x.tolist()]})
had_err = False
try:
start = datetime.now()
resp = requests.put(f'{addr}/{model}/predict', json=event_data)
if not resp.ok:
context.logger.error(f'bad function resp!!\n{resp.text}')
err_count += 1
continue
times.append((datetime.now()-start).microseconds)
except OSError as err:
context.logger.error(f'error in request, data:{event_data}, error: {err}')
err_count += 1
continue
y_resp = resp.json()[0]
if y == y_resp:
match += 1
context.log_result('total_tests', count)
context.log_result('errors', err_count)
context.log_result('match', match)
if count - err_count > 0:
times_arr = np.array(times)
context.log_result('avg_latency', int(np.mean(times_arr)))
context.log_result('min_latency', int(np.amin(times_arr)))
context.log_result('max_latency', int(np.amax(times_arr)))
chart = ChartArtifact('latency', header=['Test', 'Latency (microsec)'])
for i in range(len(times)):
chart.add_row([i+1, int(times[i])])
context.log_artifact(chart)
context.logger.info(f'run {count} tests, {err_count} errors and {match} match expected value')
if err_count:
raise ValueError(f'failed on {err_count} tests of {count}')
if match_err and match != count:
raise ValueError(f'only {match} results match out of {count}')
# nuclio: end-code
# marks the end of a code section
Deploy model server for testing#
import mlrun
project_name = 'sk-project'
MODEL_PATH = 'https://s3.wasabisys.com/iguazio/models/iris/model.pkl'
artifact_path = mlrun.set_environment(api_path = 'http://mlrun-api:8080',
artifact_path = os.path.abspath('./'))
# import model server function from hub
fn = mlrun.import_function('hub://model_server')
fn.add_model("mymodel", MODEL_PATH)
address = fn.deploy()
> 2020-10-28 16:43:54,679 [warning] warning!, server (0.5.3-rc1) and client (0.5.2) ver dont match
> 2020-10-28 16:43:55,002 [info] deploy started
[nuclio] 2020-10-28 16:45:17,274 (info) Build complete
[nuclio] 2020-10-28 16:45:22,363 done updating default-model-server, function address: default-tenant.app.dsteam.iguazio-cd1.com:30150
> 2020-10-28 16:45:22,369 [warning] warning!, server (0.5.3-rc1) and client (0.5.2) ver dont match
user_name = os.getenv('V3IO_USERNAME')
fn.apply(mlrun.mount_v3io())
fn.set_envs({'SERVING_MODEL_iris_dataset_v1': MODEL_PATH,
'INFERENCE_STREAM': 'users/{}/tststream'.format(user_name)})
address = fn.deploy()
> 2020-10-28 16:45:22,418 [info] deploy started
[nuclio] 2020-10-28 16:45:25,272 (info) Build complete
[nuclio] 2020-10-28 16:45:34,868 done updating default-model-server, function address: default-tenant.app.dsteam.iguazio-cd1.com:30150
Run model server tester locally#
# run the function locally
DATA_PATH = 'https://s3.wasabisys.com/iguazio/data/iris/iris_dataset.csv'
gen = mlrun.run_local(name='model_server_tester',
handler=model_server_tester,
params={'addr': address, 'model': 'mymodel'},
inputs={'table': DATA_PATH},
project=project_name,
artifact_path=os.path.join(artifact_path, 'data'))
> 2020-10-28 16:45:34,916 [warning] warning!, server (0.5.3-rc1) and client (0.5.2) ver dont match
> 2020-10-28 16:45:34,916 [info] starting run model_server_tester uid=c84fdd4dfacd447dbe417d709f5983f0 -> http://mlrun-api:8080
> 2020-10-28 16:45:34,972 [warning] warning!, server (0.5.3-rc1) and client (0.5.2) ver dont match
> 2020-10-28 16:45:35,264 [info] testing with dataset against http://default-tenant.app.dsteam.iguazio-cd1.com:30150, model: mymodel
> 2020-10-28 16:45:35,967 [info] run 20 tests, 0 errors and 6 match expected value
| project | uid | iter | start | state | name | labels | inputs | parameters | results | artifacts |
|---|---|---|---|---|---|---|---|---|---|---|
| sk-project | 0 | Oct 28 16:45:34 | completed | model_server_tester | v3io_user=admin kind=handler owner=admin host=jupyter-d87678b84-n4lcf |
table |
addr=http://default-tenant.app.dsteam.iguazio-cd1.com:30150 model=mymodel |
total_tests=20 errors=0 match=6 avg_latency=32619 min_latency=26263 max_latency=110524 |
latency |
to track results use .show() or .logs() or in CLI:
!mlrun get run c84fdd4dfacd447dbe417d709f5983f0 --project sk-project , !mlrun logs c84fdd4dfacd447dbe417d709f5983f0 --project sk-project
> 2020-10-28 16:45:36,042 [info] run executed, status=completed
Save#
test_func = mlrun.code_to_function(name='model_server_tester',
kind='job',
handler='model_server_tester',
description='test model servers',
categories=["ml": "test"],
labels={"author": "yaronh"})
test_func.export('function.yaml')
> 2020-10-28 16:45:43,591 [info] function spec saved to path: function.yaml
<mlrun.runtimes.kubejob.KubejobRuntime at 0x7fde772f2f50>
Run remotely#
test_func.run(mlrun.NewTask(name='model_server_tester',
handler=model_server_tester,
params={'addr': address, 'model': 'mymodel'},
inputs={'table': DATA_PATH},
project=project_name,
artifact_path=os.path.join(artifact_path, 'data')))
> 2020-10-28 16:45:43,605 [warning] warning!, server (0.5.3-rc1) and client (0.5.2) ver dont match
> 2020-10-28 16:45:43,606 [info] starting run model_server_tester uid=73cb7da1bfeb4b50afb29bce40a9c861 -> http://mlrun-api:8080
> 2020-10-28 16:45:43,738 [info] Job is running in the background, pod: model-server-tester-btgvj
| project | uid | iter | start | state | name | labels | inputs | parameters | results | artifacts |
|---|---|---|---|---|---|---|---|---|---|---|
| sk-project | 0 | Oct 28 16:45:43 | running | model_server_tester | v3io_user=admin kind=job owner=admin |
table |
addr=http://default-tenant.app.dsteam.iguazio-cd1.com:30150 model=mymodel |
to track results use .show() or .logs() or in CLI:
!mlrun get run 73cb7da1bfeb4b50afb29bce40a9c861 --project sk-project , !mlrun logs 73cb7da1bfeb4b50afb29bce40a9c861 --project sk-project
> 2020-10-28 16:45:43,790 [info] run executed, status=running
<mlrun.model.RunObject at 0x7fdeb84930d0>