#!/usr/bin/env python
#
# This script demonstrates the idea of using the anomaly detector to remove anomalous points
# to improve the ability of a predictive model to generalize. 

import bigml
from bigml.api import BigML

api = BigML()

print "Uploading source"
source = api.create_source("/Users/petersp/Desktop/CSVs/diabetes.csv")

print "Waiting for source"
api.ok(source)

print "Creating dataset"
dataset = api.create_dataset(source)

print "Waiting for dataset"
api.ok(dataset)

print "Creating training set"
train_set = api.create_dataset(dataset, { 
    "sample_rate": 0.80, 
    "out_of_bag": False, 
    "seed": "bigml",
    "name": "Diabetes Training Set"
})

print "Creating test set"
test_set = api.create_dataset(dataset, { 
    "sample_rate": 0.80, 
    "out_of_bag": True, 
    "seed": "bigml",
    "name": "Diabetes Test Set"
})

print "Waiting for training set"
api.ok(train_set)

print "Creating Anomaly Detector"
anomaly = api.create_anomaly(train_set, { 
    "name": "Diabetes Anomaly" 
})

print "Creating ALL model"
model_all = api.create_model(train_set, { 
    "name": "Diabetes Model All"
})

print "Waiting for ALL model"
api.ok(model_all)

print "Creating ALL evaluation"
eval_all = api.create_evaluation(model_all, test_set, {
    "name": "Diabetes Eval All"
})

print "Waiting for anomaly detector"
api.ok(anomaly)

print "Creating batch anomaly score"
anomaly_scores = api.create_batch_anomaly_score(anomaly, train_set, { 
    "all_fields": True, 
    "header": True, 
    "output_dataset": True,
    "name": "Diabetes Anomaly Scores"
})

print "Waiting for batch anomaly score"
api.ok(anomaly_scores)

print "Fetching batch anomaly score"
anomaly_scores = api.get_batch_anomaly_score(anomaly_scores)

print "Fetching batch anomaly output_dataset"
anomaly_dataset = api.get_dataset(anomaly_scores['object']['output_dataset_resource'])

print "Creating CLEAN dataset"
dataset_clean = api.create_dataset(anomaly_dataset, { 
    "json_filter": [ "<", [ "field", "score" ], 0.60 ], 
    "excluded_fields": [ "score" ],
    "name": "Diabetes Dataset Clean"
})

print "Waiting for CLEAN dataset"
api.ok(dataset_clean)

print "Creating CLEAN model"
model_clean = api.create_model(dataset_clean, {
    "name": "Diabetes Model Clean"
})

print "Waiing for CLEAN model"
api.ok(model_clean)

print "Creating CLEAN eval"
eval_clean = api.create_evaluation(model_clean, test_set, {
    "name": "Diabetes Eval Clean"
})