diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..9760c544591a73c89c654f56f39993d8f6c6bbb9
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,6 @@
+.ipynb_checkpoints
+.idea
+data
+__pycache__
+*.pyc
+*.csv
\ No newline at end of file
diff --git a/CodaLabPackages/.gitignore b/CodaLabPackages/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CodaLabPackages/CSEDMDC.png b/CodaLabPackages/CSEDMDC.png
new file mode 100644
index 0000000000000000000000000000000000000000..f8f56405984bf555d704791a6a4b06ee4cf13c65
Binary files /dev/null and b/CodaLabPackages/CSEDMDC.png differ
diff --git a/CodaLabPackages/Track1Package/competition.yaml b/CodaLabPackages/Track1Package/competition.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..58b4506b399334e85b139f5437e84310172dfac4
--- /dev/null
+++ b/CodaLabPackages/Track1Package/competition.yaml
@@ -0,0 +1,61 @@
+title: 2th CSEDM Challenge -- Student Struggling Prediction
+description: Tracking student programming traces and predicting their struggles in late problems.
+image: csedm image.jpeg
+has_registration: True
+html:
+ overview: overview.html
+ evaluation: evaluation.html
+ terms: terms_and_conditions.html
+ data: data.html
+phases:
+ 1:
+ phasenumber: 1
+ label: "Practice"
+ start_date: 2021-06-28
+ max_submissions: 100
+ scoring_program: program.zip
+ reference_data: reference1.zip
+ starting_kit: submission1.zip
+ 2:
+ phasenumber: 2
+ label: "Cross-Semester"
+ start_date: 2021-06-28
+ max_submissions: 100
+ scoring_program: program.zip
+ reference_data: reference2.zip
+ starting_kit: submission2.zip
+ 3:
+ phasenumber: 3
+ label: "Within-Semester"
+ start_date: 2022-02-07
+ max_submissions: 100
+ scoring_program: program.zip
+ reference_data: reference3.zip
+ starting_kit: submission3.zip
+leaderboard:
+ leaderboards:
+ RESULTS: &RESULTS
+ label: Results
+ rank: 1
+ columns:
+ AUC:
+ leaderboard: *RESULTS
+ label: AUC
+ rank: 1
+ numeric_format: 4
+ MACRO_F1:
+ leaderboard: *RESULTS
+ label: Macro F1
+ rank: 2
+ numeric_format: 4
+ POSITIVE_F1:
+ leaderboard: *RESULTS
+ label: Positive F1
+ rank: 3
+ numeric_format: 4
+ ACC:
+ leaderboard: *RESULTS
+ label: Accuracy
+ rank: 4
+ numeric_format: 4
+
diff --git a/CodaLabPackages/Track1Package/csedm image.jpeg b/CodaLabPackages/Track1Package/csedm image.jpeg
new file mode 100644
index 0000000000000000000000000000000000000000..e118ecaa874819b4fcdf8fda03182a3a2065b6b0
Binary files /dev/null and b/CodaLabPackages/Track1Package/csedm image.jpeg differ
diff --git a/CodaLabPackages/Track1Package/data.html b/CodaLabPackages/Track1Package/data.html
new file mode 100644
index 0000000000000000000000000000000000000000..c4464e8dbf6a91c844302d7455a9a0ec41fe8814
--- /dev/null
+++ b/CodaLabPackages/Track1Package/data.html
@@ -0,0 +1,3 @@
+<p>
+ This is the data for the competition. It is to be used responsibly.
+</p>
\ No newline at end of file
diff --git a/CodaLabPackages/Track1Package/evaluation.html b/CodaLabPackages/Track1Package/evaluation.html
new file mode 100644
index 0000000000000000000000000000000000000000..a2705a1a3d04dcdd4cbc574b7d2ec634bd268d0b
--- /dev/null
+++ b/CodaLabPackages/Track1Package/evaluation.html
@@ -0,0 +1,5 @@
+<H3>Evaluation Criteria</H3>
+
+<p>
+ This is the page that tells you how competition submissions will be evaluated and scored.
+</p>
\ No newline at end of file
diff --git a/CodaLabPackages/Track1Package/overview.html b/CodaLabPackages/Track1Package/overview.html
new file mode 100644
index 0000000000000000000000000000000000000000..6fa77c7c2f1a7cb636cbf6805820b042a89749c2
--- /dev/null
+++ b/CodaLabPackages/Track1Package/overview.html
@@ -0,0 +1,5 @@
+<H3>Welcome!</H3>
+
+<p>
+ This is an example competition.
+</p>
\ No newline at end of file
diff --git a/CodaLabPackages/Track1Package/program.zip b/CodaLabPackages/Track1Package/program.zip
new file mode 100644
index 0000000000000000000000000000000000000000..d98e2b65ace7c8958e71581ff0e8dcbcf27d0169
Binary files /dev/null and b/CodaLabPackages/Track1Package/program.zip differ
diff --git a/CodaLabPackages/Track1Package/program/evaluate.py b/CodaLabPackages/Track1Package/program/evaluate.py
new file mode 100644
index 0000000000000000000000000000000000000000..c081a03eba00e02043b111cea6d7ef2122e929c8
--- /dev/null
+++ b/CodaLabPackages/Track1Package/program/evaluate.py
@@ -0,0 +1,77 @@
+import sys, os
+from sklearn.metrics import f1_score, accuracy_score, roc_auc_score
+import pandas as pd
+import numpy as np
+import time
+
+
+print("vamos a empezar")
+time.sleep(2)
+
+input_dir = sys.argv[1]
+output_dir = sys.argv[2]
+
+print("directorios de entrada y salida ingresados")
+time.sleep(2)
+
+submit_dir = os.path.join(input_dir, 'res')
+truth_dir = os.path.join(input_dir, 'ref')
+
+print("join de los directorios para res y ref realizados")
+time.sleep(2)
+
+if not os.path.isdir(submit_dir):
+ print("%s doesn't exist" % submit_dir)
+else:
+ print("directorio input + res (submit_dir) si existe")
+ time.sleep(2)
+
+if os.path.isdir(submit_dir) and os.path.isdir(truth_dir):
+
+ print("vamos a bien, segundo if")
+ time.sleep(2)
+
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+ else:
+ print("si existe directorio de salida")
+ time.sleep(2)
+
+
+
+ output_filename = os.path.join(output_dir, 'scores.txt')
+ output_file = open(output_filename, 'wb')
+ print("archivo score.txt creado correctamente")
+ time.sleep(2)
+
+ true_csv = pd.read_csv(os.path.join(truth_dir, "truth.csv"))
+ pred_csv = pd.read_csv(os.path.join(submit_dir, "predictions.csv"))
+
+ assert len(true_csv) == len(pred_csv), "Submission with wrong number of entries: Should be " + str(len(true_csv))
+ assert 'SubjectID' in pred_csv.columns and 'ProblemID' in pred_csv.columns and 'Label' in pred_csv.columns, \
+ "Submission columns should be: SubjectID, ProblemID, Label"
+ assert set(true_csv['SubjectID']) == set(pred_csv['SubjectID']), "Submission SubjectIDs do not match."
+ assert set(true_csv['ProblemID']) == set(pred_csv['ProblemID']), "Submission ProblemIDs do not match."
+ assert set(pred_csv["Label"]) != set([False,True]), \
+ "Submission should include probabilities rather than binary results."
+
+ df = true_csv.set_index(['SubjectID','ProblemID']).join(pred_csv.set_index(['SubjectID','ProblemID']), rsuffix="ScorePrediction")
+ df["LabelPrediction"] = df["LabelScorePrediction"] > 0.5
+
+ f1_negative = f1_score(1-df["Label"], 1-df["LabelPrediction"])
+ f1_positive = f1_score(df["Label"], df["LabelPrediction"])
+ f1 = (f1_negative + f1_positive)/2
+ acc = accuracy_score(df["Label"], df["LabelPrediction"])
+ auc = roc_auc_score(df["Label"], df["LabelScorePrediction"])
+
+
+ print("MACRO F1: %f" % f1)
+ print("POSITIVE F1: %f" % f1_positive)
+ print("ACC: %f" % acc)
+ print("AUC: %f" % auc)
+
+ output_file.write(b"MACRO_F1: %f \n" % f1)
+ output_file.write(b"POSITIVE_F1: %f \n" % f1_positive)
+ output_file.write(b"ACC: %f \n" % acc)
+ output_file.write(b"AUC: %f \n" % auc)
+ output_file.close()
diff --git a/CodaLabPackages/Track1Package/program/metadata b/CodaLabPackages/Track1Package/program/metadata
new file mode 100644
index 0000000000000000000000000000000000000000..9691767d67c02ee03e327f01db36c09765e19310
--- /dev/null
+++ b/CodaLabPackages/Track1Package/program/metadata
@@ -0,0 +1,2 @@
+command: python $program/evaluate.py $input $output
+description: Example competition evaluation program.
diff --git a/CodaLabPackages/Track1Package/program/readme.md b/CodaLabPackages/Track1Package/program/readme.md
new file mode 100644
index 0000000000000000000000000000000000000000..45f5468da17f9fa851dc08fdd5ba6397d061fa3b
--- /dev/null
+++ b/CodaLabPackages/Track1Package/program/readme.md
@@ -0,0 +1,8 @@
+Building an evaluation program that works with CodaLab
+
+This example uses python. It assumes python is installed on the codalab worker machines.
+
+evaluate.py - is an example that loads a single value from each of the gold files, looks for a corresponding submission, and finds the difference.
+metadata - this is a file that lists the contents of the program.zip bundle for the CodaLab system.
+
+Once these pieces are assembled they are packages as program.zip which CodaLab can then use to evaluate the submissions for a competition.
\ No newline at end of file
diff --git a/CodaLabPackages/Track1Package/submission.zip b/CodaLabPackages/Track1Package/submission.zip
new file mode 100644
index 0000000000000000000000000000000000000000..f3a6adfe6868f6d4977baf33b7c662aa58c1e3b9
Binary files /dev/null and b/CodaLabPackages/Track1Package/submission.zip differ
diff --git a/CodaLabPackages/Track1Package/submission/scores.txt b/CodaLabPackages/Track1Package/submission/scores.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CodaLabPackages/Track1Package/terms_and_conditions.html b/CodaLabPackages/Track1Package/terms_and_conditions.html
new file mode 100644
index 0000000000000000000000000000000000000000..de518cddc92d8e60b2c90191a4913e743d0f5f04
--- /dev/null
+++ b/CodaLabPackages/Track1Package/terms_and_conditions.html
@@ -0,0 +1,5 @@
+<H3>Terms and Conditions</H3>
+
+<p>
+ This page enumerated the terms and conditions of the competition.
+</p>
\ No newline at end of file
diff --git a/CodaLabPackages/Track2Package/competition.yaml b/CodaLabPackages/Track2Package/competition.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2ffb86bb9c786b0327cca20e322b0b1b3767d5d6
--- /dev/null
+++ b/CodaLabPackages/Track2Package/competition.yaml
@@ -0,0 +1,29 @@
+title: Example Hello World Competition
+description: An example competition where submissions should output "Hello World!"
+image: csedm image.jpeg
+has_registration: True
+html:
+ overview: overview.html
+ evaluation: evaluation.html
+ terms: terms_and_conditions.html
+ data: data.html
+phases:
+ 1:
+ phasenumber: 1
+ label: "First phase"
+ start_date: 2013-06-30
+ max_submissions: 100
+ scoring_program: program.zip
+ reference_data: reference.zip
+ starting_kit: submission.zip
+leaderboard:
+ leaderboards:
+ RESULTS: &RESULTS
+ label: Results
+ rank: 1
+ columns:
+ correct:
+ leaderboard: *RESULTS
+ label: correct
+ rank: 1
+ numeric_format: 1
diff --git a/CodaLabPackages/Track2Package/csedm image.jpeg b/CodaLabPackages/Track2Package/csedm image.jpeg
new file mode 100644
index 0000000000000000000000000000000000000000..e118ecaa874819b4fcdf8fda03182a3a2065b6b0
Binary files /dev/null and b/CodaLabPackages/Track2Package/csedm image.jpeg differ
diff --git a/CodaLabPackages/Track2Package/data.html b/CodaLabPackages/Track2Package/data.html
new file mode 100644
index 0000000000000000000000000000000000000000..c4464e8dbf6a91c844302d7455a9a0ec41fe8814
--- /dev/null
+++ b/CodaLabPackages/Track2Package/data.html
@@ -0,0 +1,3 @@
+<p>
+ This is the data for the competition. It is to be used responsibly.
+</p>
\ No newline at end of file
diff --git a/CodaLabPackages/Track2Package/evaluation.html b/CodaLabPackages/Track2Package/evaluation.html
new file mode 100644
index 0000000000000000000000000000000000000000..a2705a1a3d04dcdd4cbc574b7d2ec634bd268d0b
--- /dev/null
+++ b/CodaLabPackages/Track2Package/evaluation.html
@@ -0,0 +1,5 @@
+<H3>Evaluation Criteria</H3>
+
+<p>
+ This is the page that tells you how competition submissions will be evaluated and scored.
+</p>
\ No newline at end of file
diff --git a/CodaLabPackages/Track2Package/overview.html b/CodaLabPackages/Track2Package/overview.html
new file mode 100644
index 0000000000000000000000000000000000000000..6fa77c7c2f1a7cb636cbf6805820b042a89749c2
--- /dev/null
+++ b/CodaLabPackages/Track2Package/overview.html
@@ -0,0 +1,5 @@
+<H3>Welcome!</H3>
+
+<p>
+ This is an example competition.
+</p>
\ No newline at end of file
diff --git a/CodaLabPackages/Track2Package/program.zip b/CodaLabPackages/Track2Package/program.zip
new file mode 100644
index 0000000000000000000000000000000000000000..f809293589691869182bcf4da02252552a01631a
Binary files /dev/null and b/CodaLabPackages/Track2Package/program.zip differ
diff --git a/CodaLabPackages/Track2Package/program/evaluate.py b/CodaLabPackages/Track2Package/program/evaluate.py
new file mode 100644
index 0000000000000000000000000000000000000000..673f3a265507cf931b3d1b297345da2ab8630623
--- /dev/null
+++ b/CodaLabPackages/Track2Package/program/evaluate.py
@@ -0,0 +1,39 @@
+import sys, os
+from sklearn.metrics import mean_squared_error
+import pandas as pd
+
+
+input_dir = sys.argv[1]
+output_dir = sys.argv[2]
+
+submit_dir = os.path.join(input_dir, 'res')
+truth_dir = os.path.join(input_dir, 'ref')
+
+if not os.path.isdir(submit_dir):
+ print("%s doesn't exist" % submit_dir)
+
+if os.path.isdir(submit_dir) and os.path.isdir(truth_dir):
+
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+
+ output_filename = os.path.join(output_dir, 'scores.txt')
+ output_file = open(output_filename, 'wb')
+
+ true_csv = pd.read_csv(os.path.join(truth_dir, "truth.csv"))
+ pred_csv = pd.read_csv(os.path.join(submit_dir, "predictions.csv"))
+
+ assert len(true_csv) == len(pred_csv), "Submission with wrong number of entries: Should be " + str(len(true_csv))
+ assert 'SubjectID' in pred_csv.columns and 'X-Grade' in pred_csv.columns, \
+ "Submission columns should be: SubjectID, X-Grade"
+ assert set(true_csv['SubjectID']) == set(pred_csv['SubjectID']), "Submission SubjectIDs do not match."
+ assert set(pred_csv["X-Grade"]) != set(
+ [False, True]), "Submission should be a continuous grade prediction, not binary."
+
+ df = true_csv.set_index('SubjectID').join(pred_csv.set_index('SubjectID'), rsuffix="Prediction")
+
+ mse = mean_squared_error(df["X-Grade"], df["X-GradePrediction"])
+ print(mse)
+
+ output_file.write(b"MSE: %f" % mse)
+ output_file.close()
diff --git a/CodaLabPackages/Track2Package/program/metadata b/CodaLabPackages/Track2Package/program/metadata
new file mode 100644
index 0000000000000000000000000000000000000000..9691767d67c02ee03e327f01db36c09765e19310
--- /dev/null
+++ b/CodaLabPackages/Track2Package/program/metadata
@@ -0,0 +1,2 @@
+command: python $program/evaluate.py $input $output
+description: Example competition evaluation program.
diff --git a/CodaLabPackages/Track2Package/program/readme.md b/CodaLabPackages/Track2Package/program/readme.md
new file mode 100644
index 0000000000000000000000000000000000000000..45f5468da17f9fa851dc08fdd5ba6397d061fa3b
--- /dev/null
+++ b/CodaLabPackages/Track2Package/program/readme.md
@@ -0,0 +1,8 @@
+Building an evaluation program that works with CodaLab
+
+This example uses python. It assumes python is installed on the codalab worker machines.
+
+evaluate.py - is an example that loads a single value from each of the gold files, looks for a corresponding submission, and finds the difference.
+metadata - this is a file that lists the contents of the program.zip bundle for the CodaLab system.
+
+Once these pieces are assembled they are packages as program.zip which CodaLab can then use to evaluate the submissions for a competition.
\ No newline at end of file
diff --git a/CodaLabPackages/Track2Package/submission.zip b/CodaLabPackages/Track2Package/submission.zip
new file mode 100644
index 0000000000000000000000000000000000000000..48dec4f2d0189fa4669eab4c512fa95017b83194
Binary files /dev/null and b/CodaLabPackages/Track2Package/submission.zip differ
diff --git a/CodaLabPackages/Track2Package/terms_and_conditions.html b/CodaLabPackages/Track2Package/terms_and_conditions.html
new file mode 100644
index 0000000000000000000000000000000000000000..de518cddc92d8e60b2c90191a4913e743d0f5f04
--- /dev/null
+++ b/CodaLabPackages/Track2Package/terms_and_conditions.html
@@ -0,0 +1,5 @@
+<H3>Terms and Conditions</H3>
+
+<p>
+ This page enumerated the terms and conditions of the competition.
+</p>
\ No newline at end of file
diff --git a/ProgSnap2.py b/ProgSnap2.py
new file mode 100644
index 0000000000000000000000000000000000000000..f639e6e8386edcab608240158d86cc56df141b7d
--- /dev/null
+++ b/ProgSnap2.py
@@ -0,0 +1,205 @@
+import pandas as pd
+import os
+from os import path
+
+
+class PS2:
+ """ A class holding constants used to get columns of a PS2 dataset
+ """
+
+ Order = 'Order'
+ SubjectID = 'SubjectID'
+ ToolInstances = 'ToolInstances'
+ ServerTimestamp = 'ServerTimestamp'
+ ServerTimezone = 'ServerTimezone'
+ CourseID = 'CourseID'
+ CourseSectionID = 'CourseSectionID'
+ AssignmentID = 'AssignmentID'
+ ProblemID = 'ProblemID'
+ Attempt = 'Attempt'
+ CodeStateID = 'CodeStateID'
+ IsEventOrderingConsistent = 'IsEventOrderingConsistent'
+ EventType = 'EventType'
+ Score = 'Score'
+ CompileResult = 'CompileResult'
+ CompileMessageType = 'CompileMessageType'
+ CompileMessageData = 'CompileMessageData'
+ EventID = 'EventID'
+ ParentEventID = 'ParentEventID'
+ SourceLocation = 'SourceLocation'
+ Code = 'Code'
+
+ Version = 'Version'
+ IsEventOrderingConsistent = 'IsEventOrderingConsistent'
+ EventOrderScope = 'EventOrderScope'
+ EventOrderScopeColumns = 'EventOrderScopeColumns'
+ CodeStateRepresentation = 'CodeStateRepresentation'
+
+
+class ProgSnap2Dataset:
+
+ MAIN_TABLE_FILE = 'MainTable.csv'
+ METADATA_TABLE_FILE = 'DatasetMetadata.csv'
+ LINK_TABLE_DIR = 'LinkTables'
+ CODE_STATES_DIR = 'CodeStates'
+ CODE_STATES_TABLE_FILE = os.path.join(CODE_STATES_DIR, 'CodeStates.csv')
+
+ def __init__(self, directory):
+ self.directory = directory
+ self.main_table = None
+ self.metadata_table = None
+ self.code_states_table = None
+
+ def path(self, local_path):
+ return path.join(self.directory, local_path)
+
+ def get_main_table(self):
+ """ Returns a Pandas DataFrame with the main event table for this dataset
+ """
+ if self.main_table is None:
+ self.main_table = pd.read_csv(self.path(ProgSnap2Dataset.MAIN_TABLE_FILE))
+ if self.get_metadata_property(PS2.IsEventOrderingConsistent):
+ order_scope = self.get_metadata_property(PS2.EventOrderScope)
+ if order_scope == 'Global':
+ # If the table is globally ordered, sort it
+ self.main_table.sort_values(by=[PS2.Order], inplace=True)
+ elif order_scope == 'Restricted':
+ # If restricted ordered, sort first by grouping columns, then by order
+ order_columns = self.get_metadata_property(PS2.EventOrderScopeColumns)
+ if order_columns is None or len(order_columns) == 0:
+ raise Exception('EventOrderScope is restricted by no EventOrderScopeColumns given')
+ columns = order_columns.split(';')
+ columns.append('Order')
+ # The result is that _within_ these groups, events are ordered
+ self.main_table.sort_values(by=columns, inplace=True)
+ return self.main_table.copy()
+
+ def set_main_table(self, main_table):
+ """ Overwrites the main table loaded from the file with the provided table.
+ This this table will be used for future operations, including copying the dataset.
+ """
+ self.main_table = main_table.copy()
+
+ def get_code_states_table(self):
+ """ Returns a Pandas DataFrame with the code states table form this dataset
+ """
+ if self.code_states_table is None:
+ self.code_states_table = pd.read_csv(self.path(ProgSnap2Dataset.CODE_STATES_TABLE_FILE))
+ return self.code_states_table.copy()
+
+ def get_metadata_property(self, property):
+ """ Returns the value of a given metadata property in the metadata table
+ """
+ if self.metadata_table is None:
+ self.metadata_table = pd.read_csv(self.path(ProgSnap2Dataset.METADATA_TABLE_FILE))
+
+ values = self.metadata_table[self.metadata_table['Property'] == property]['Value']
+ if len(values) == 1:
+ return values.iloc[0]
+ if len(values) > 1:
+ raise Exception('Multiple values for property: ' + property)
+
+ # Default return values as of V6
+ if property == PS2.IsEventOrderingConsistent:
+ return False
+ if property == PS2.EventOrderScope:
+ return 'None'
+ if property == PS2.EventOrderScopeColumns:
+ return ''
+
+ return None
+
+ def __link_table_path(self):
+ return self.path(ProgSnap2Dataset.LINK_TABLE_DIR)
+
+ def list_link_tables(self):
+ """ Returns a list of the link tables in this dataset, which can be loaded with load_link_table
+ """
+ path = self.__link_table_path()
+ dirs = os.listdir(path)
+ return [f for f in dirs if os.path.isfile(os.path.join(path, f)) and f.endswith('.csv')]
+
+ def load_link_table(self, link_table):
+ """ Returns a Pandas DataFrame with the link table with the given name
+ :param link_table: The link table nme or file
+ """
+ if not link_table.endswith('.csv'):
+ link_table += '.csv'
+ return pd.read_csv(path.join(self.__link_table_path(), link_table))
+
+ def drop_main_table_column(self, column):
+ self.get_main_table()
+ self.main_table.drop(column, axis=1, inplace=True)
+
+ def save_subset(self, path, main_table_filterer, copy_link_tables=True):
+ os.makedirs(os.path.join(path, ProgSnap2Dataset.CODE_STATES_DIR), exist_ok=True)
+ main_table = main_table_filterer(self.get_main_table())
+ main_table.to_csv(os.path.join(path, ProgSnap2Dataset.MAIN_TABLE_FILE), index=False)
+ code_state_ids = main_table[PS2.CodeStateID].unique()
+ code_states = self.get_code_states_table()
+ code_states = code_states[code_states[PS2.CodeStateID].isin(code_state_ids)]
+ code_states.to_csv(os.path.join(path, ProgSnap2Dataset.CODE_STATES_DIR, 'CodeStates.csv'), index=False)
+ self.metadata_table.to_csv(os.path.join(path, ProgSnap2Dataset.METADATA_TABLE_FILE), index=False)
+
+ if not copy_link_tables:
+ return
+
+ os.makedirs(os.path.join(path, ProgSnap2Dataset.LINK_TABLE_DIR), exist_ok=True)
+
+ def indexify(x):
+ return tuple(x) if len(x) > 1 else x[0]
+
+ for link_table_name in self.list_link_tables():
+ link_table = self.load_link_table(link_table_name)
+ columns = [col for col in link_table.columns if col.endswith('ID') and col in main_table.columns]
+ distinct_ids = main_table.groupby(columns).apply(lambda x: True)
+ # TODO: Still need to test this with multi-ID link tables
+ to_keep = [indexify(list(row)) in distinct_ids for index, row in link_table[columns].iterrows()]
+ filtered_link_table = link_table[to_keep]
+ filtered_link_table.to_csv(os.path.join(path, ProgSnap2Dataset.LINK_TABLE_DIR, link_table_name), index=False)
+
+
+
+ @staticmethod
+ def __to_one(lst, error):
+ if len(lst) == 0:
+ return None
+ if len(lst) > 1:
+ raise Exception(error or 'Should have only one result!')
+ return lst.iloc[0]
+
+ def get_code_for_id(self, code_state_id):
+ if code_state_id is None:
+ return None
+ code_states = self.get_code_states_table()
+ code = code_states[code_states[PS2.CodeStateID] == code_state_id][PS2.Code]
+ return ProgSnap2Dataset.__to_one(code, 'Multiple code states match that ID.')
+
+ def get_code_for_event_id(self, row_id):
+ events = self.get_main_table()
+ code_state_ids = events[events[PS2.EventID == row_id]][PS2.CodeStateID]
+ code_state_id = ProgSnap2Dataset.__to_one(code_state_ids, 'Multiple rows match that ID.')
+ return self.get_code_for_id(code_state_id)
+
+ def get_subject_ids(self):
+ events = self.get_main_table()
+ return events[PS2.SubjectID].unique()
+
+ def get_problem_ids(self):
+ events = self.get_main_table()
+ return events[PS2.ProblemID].unique()
+
+ def get_trace(self, subject_id, problem_id):
+ events = self.get_main_table()
+ rows = events[(events[PS2.SubjectID] == subject_id) & (events[PS2.ProblemID] == problem_id)]
+ ids = rows[PS2.CodeStateID].unique()
+ return [self.get_code_for_id(code_state_id) for code_state_id in ids]
+
+
+if __name__ == '__main__':
+ data = ProgSnap2Dataset('data/CodeWorkout/S19')
+ # for code in data.get_trace('4d230b683bf9840553ae57f4acc96e81', 32):
+ # print(code)
+ # print('-------')
+
+ data.save_subset('data/test/CopyA', lambda df: df[df[PS2.SubjectID].str.startswith('a')])
diff --git a/analisis de la matriz de x_train.png b/analisis de la matriz de x_train.png
new file mode 100644
index 0000000000000000000000000000000000000000..b8c60e58096fc9990491c957e866ec861a0d23c8
Binary files /dev/null and b/analisis de la matriz de x_train.png differ
diff --git a/code_feature_model.ipynb b/code_feature_model.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..d200f7d3c6f1edac75e7d9f8d18f53ba846edbe1
--- /dev/null
+++ b/code_feature_model.ipynb
@@ -0,0 +1,1367 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "from ProgSnap2 import ProgSnap2Dataset\n",
+ "from ProgSnap2 import PS2\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.preprocessing import OneHotEncoder\n",
+ "import numpy as np\n",
+ "import os\n",
+ "from os import path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "semester = 'S19'\n",
+ "BASE_PATH = os.path.join('data', 'Release', semester)\n",
+ "TRAIN_PATH = os.path.join(BASE_PATH, 'Train')\n",
+ "TEST_PATH = os.path.join(BASE_PATH, 'Test')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_ps2 = ProgSnap2Dataset(os.path.join(TRAIN_PATH, 'Data')) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Attempts</th>\n",
+ " <th>CorrectEventually</th>\n",
+ " <th>Label</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>3</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>5</td>\n",
+ " <td>3</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>12</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>13</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Attempts \\\n",
+ "0 04c32d4d95425f73b3a1d6502aed4d48 439.0 1 1 \n",
+ "1 04c32d4d95425f73b3a1d6502aed4d48 439.0 3 2 \n",
+ "2 04c32d4d95425f73b3a1d6502aed4d48 439.0 5 3 \n",
+ "3 04c32d4d95425f73b3a1d6502aed4d48 439.0 12 1 \n",
+ "4 04c32d4d95425f73b3a1d6502aed4d48 439.0 13 2 \n",
+ "\n",
+ " CorrectEventually Label \n",
+ "0 True True \n",
+ "1 True True \n",
+ "2 True True \n",
+ "3 True True \n",
+ "4 True True "
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# The early dataset will help us to feature extraction,\n",
+ "# but we're not actually predicting anything here\n",
+ "# Note: we could still use this for model training if desired.\n",
+ "early_train = pd.read_csv(os.path.join(TRAIN_PATH, 'early.csv'))\n",
+ "early_train.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Label</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>41</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>43</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>44</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>46</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>49</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Label\n",
+ "0 04c32d4d95425f73b3a1d6502aed4d48 494.0 41 False\n",
+ "1 04c32d4d95425f73b3a1d6502aed4d48 494.0 43 True\n",
+ "2 04c32d4d95425f73b3a1d6502aed4d48 494.0 44 True\n",
+ "3 04c32d4d95425f73b3a1d6502aed4d48 494.0 46 True\n",
+ "4 04c32d4d95425f73b3a1d6502aed4d48 494.0 49 True"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# The late dataset contains the problems that we're actually predicting for.\n",
+ "# The training portion of it includes labels.\n",
+ "late_train = pd.read_csv(os.path.join(TRAIN_PATH, 'late.csv'))\n",
+ "late_train.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "X_train_base = late_train.copy().drop('Label', axis=1)\n",
+ "y_train = late_train['Label'].values"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "problem_encoder = OneHotEncoder().fit(X_train_base[PS2.ProblemID].values.reshape(-1, 1))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([[1., 0., 0., ..., 0., 0., 0.],\n",
+ " [0., 1., 0., ..., 0., 0., 0.],\n",
+ " [0., 0., 1., ..., 0., 0., 0.],\n",
+ " ...,\n",
+ " [0., 0., 0., ..., 0., 0., 0.],\n",
+ " [0., 0., 0., ..., 0., 1., 0.],\n",
+ " [0., 0., 0., ..., 0., 0., 1.]])"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "problem_encoder.transform(X_train_base[PS2.ProblemID].values.reshape(-1, 1)).toarray()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_code_table(ps2_dataset):\n",
+ " events_table = ps2_dataset.get_main_table()\n",
+ " code_states = ps2_dataset.get_code_states_table()\n",
+ " runs = events_table.merge(code_states, on=PS2.CodeStateID)\n",
+ " runs = runs[runs[PS2.EventType] == 'Run.Program']\n",
+ " runs = runs[[PS2.Order, PS2.SubjectID, PS2.ProblemID, 'Code']]\n",
+ " return runs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>Order</th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Code</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>119441</td>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>1</td>\n",
+ " <td>public int sortaSum(int a, int b)\\r\\n{\\r\\n ...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>134115</td>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>3</td>\n",
+ " <td>public boolean in1To10(int n, boolean outsideM...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>134117</td>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>3</td>\n",
+ " <td>public boolean in1To10(int n, boolean outsideM...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>6</th>\n",
+ " <td>65403</td>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>5</td>\n",
+ " <td>public boolean answerCell(boolean isMorning, b...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>10</th>\n",
+ " <td>65407</td>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>5</td>\n",
+ " <td>public boolean answerCell(boolean isMorning, b...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>134497</th>\n",
+ " <td>113478</td>\n",
+ " <td>ffb72475a81de0e95b910ffad039f5c2</td>\n",
+ " <td>70</td>\n",
+ " <td>public boolean twoTwo(int[] nums)\\r\\n{\\r\\n ...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>134500</th>\n",
+ " <td>113483</td>\n",
+ " <td>ffb72475a81de0e95b910ffad039f5c2</td>\n",
+ " <td>70</td>\n",
+ " <td>public boolean twoTwo(int[] nums)\\r\\n{\\r\\n ...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>134502</th>\n",
+ " <td>108282</td>\n",
+ " <td>ffb72475a81de0e95b910ffad039f5c2</td>\n",
+ " <td>71</td>\n",
+ " <td>public boolean canBalance(int[] nums)\\r\\n{\\r\\n...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>134504</th>\n",
+ " <td>110202</td>\n",
+ " <td>ffb72475a81de0e95b910ffad039f5c2</td>\n",
+ " <td>112</td>\n",
+ " <td>public int[] seriesUp(int n)\\r\\n{\\r\\n int[]...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>134506</th>\n",
+ " <td>81273</td>\n",
+ " <td>ffb72475a81de0e95b910ffad039f5c2</td>\n",
+ " <td>118</td>\n",
+ " <td>public int[] shiftLeft(int[] nums)\\r\\n{\\r\\n ...</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>46825 rows × 4 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " Order SubjectID ProblemID \\\n",
+ "0 119441 04c32d4d95425f73b3a1d6502aed4d48 1 \n",
+ "2 134115 04c32d4d95425f73b3a1d6502aed4d48 3 \n",
+ "4 134117 04c32d4d95425f73b3a1d6502aed4d48 3 \n",
+ "6 65403 04c32d4d95425f73b3a1d6502aed4d48 5 \n",
+ "10 65407 04c32d4d95425f73b3a1d6502aed4d48 5 \n",
+ "... ... ... ... \n",
+ "134497 113478 ffb72475a81de0e95b910ffad039f5c2 70 \n",
+ "134500 113483 ffb72475a81de0e95b910ffad039f5c2 70 \n",
+ "134502 108282 ffb72475a81de0e95b910ffad039f5c2 71 \n",
+ "134504 110202 ffb72475a81de0e95b910ffad039f5c2 112 \n",
+ "134506 81273 ffb72475a81de0e95b910ffad039f5c2 118 \n",
+ "\n",
+ " Code \n",
+ "0 public int sortaSum(int a, int b)\\r\\n{\\r\\n ... \n",
+ "2 public boolean in1To10(int n, boolean outsideM... \n",
+ "4 public boolean in1To10(int n, boolean outsideM... \n",
+ "6 public boolean answerCell(boolean isMorning, b... \n",
+ "10 public boolean answerCell(boolean isMorning, b... \n",
+ "... ... \n",
+ "134497 public boolean twoTwo(int[] nums)\\r\\n{\\r\\n ... \n",
+ "134500 public boolean twoTwo(int[] nums)\\r\\n{\\r\\n ... \n",
+ "134502 public boolean canBalance(int[] nums)\\r\\n{\\r\\n... \n",
+ "134504 public int[] seriesUp(int n)\\r\\n{\\r\\n int[]... \n",
+ "134506 public int[] shiftLeft(int[] nums)\\r\\n{\\r\\n ... \n",
+ "\n",
+ "[46825 rows x 4 columns]"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "code_table_train = get_code_table(train_ps2)\n",
+ "code_table_train"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'public': 19,\n",
+ " 'int': 14,\n",
+ " 'if': 13,\n",
+ " '10': 0,\n",
+ " 'return': 21,\n",
+ " 'else': 7,\n",
+ " 'boolean': 3,\n",
+ " 'true': 28,\n",
+ " 'false': 10,\n",
+ " 'speed': 23,\n",
+ " 'string': 25,\n",
+ " 'day': 6,\n",
+ " '21': 1,\n",
+ " 'str': 24,\n",
+ " 'substring': 26,\n",
+ " 'length': 15,\n",
+ " 'num': 17,\n",
+ " 'small': 22,\n",
+ " 'big': 2,\n",
+ " 'goal': 12,\n",
+ " 'for': 11,\n",
+ " 'end': 8,\n",
+ " 'equals': 9,\n",
+ " 'word': 29,\n",
+ " 'new': 16,\n",
+ " 'count': 5,\n",
+ " 'charat': 4,\n",
+ " 'nums': 18,\n",
+ " 'sum': 27,\n",
+ " 'result': 20}"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# We want to find a consistent, common vocabulary across all problems\n",
+ "# so we first build our vocabulary for all code submissions\n",
+ "\n",
+ "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+ "\n",
+ "# Note this approach is _very_ naive, since it's using NLP assumptions\n",
+ "# about tokenizing, among other things, but it is good enough for a demonstration.\n",
+ "code_vectorizer = TfidfVectorizer(max_features=30)\n",
+ "code_vectorizer.fit(code_table_train['Code'])\n",
+ "top_vocab = code_vectorizer.vocabulary_\n",
+ "top_vocab"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "50"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# We want to create a separate encoder for each problem, since the\n",
+ "# \"document frequency\" part of TF-IDF should be calibrated separately\n",
+ "# for each problem.\n",
+ "code_problem_encoders = {}\n",
+ "def create_encoder(rows):\n",
+ " code = rows['Code']\n",
+ " problem_id = rows[PS2.ProblemID].iloc[0]\n",
+ " code_vectorizer = TfidfVectorizer(vocabulary=top_vocab)\n",
+ " code_vectorizer.fit(code)\n",
+ " code_problem_encoders[problem_id] = code_vectorizer\n",
+ " \n",
+ "code_table_train.groupby(PS2.ProblemID).apply(create_encoder)\n",
+ "len(code_problem_encoders)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "public int sortaSum(int a, int b)\r\n",
+ "{\r\n",
+ " if (a + b >= 10)\r\n",
+ " {\r\n",
+ " if (a + b <= 19)\r\n",
+ " {\r\n",
+ " return 20;\r\n",
+ " }\r\n",
+ " return a + b;\r\n",
+ " }\r\n",
+ " else\r\n",
+ " {\r\n",
+ " return a + b;\r\n",
+ " }\r\n",
+ "}\r\n",
+ "\n",
+ " (0, 21)\t0.5929619322827931\n",
+ " (0, 19)\t0.19320378937452193\n",
+ " (0, 14)\t0.5796113681235658\n",
+ " (0, 13)\t0.39681148818701084\n",
+ " (0, 7)\t0.2744965584355051\n",
+ " (0, 0)\t0.20569731847202877\n"
+ ]
+ }
+ ],
+ "source": [
+ "test_code = code_table_train['Code'].iloc[0]\n",
+ "print(test_code)\n",
+ "print(code_problem_encoders[1].transform([test_code]))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def extract_instance_features(instance, early_df):\n",
+ " instance = instance.copy()\n",
+ " subject_id = instance[PS2.SubjectID]\n",
+ " early_problems = early_df[early_df[PS2.SubjectID] == subject_id]\n",
+ " # Extract very naive features about the student\n",
+ " # (without respect to the problem bring predicted)\n",
+ " # Number of early problems attempted\n",
+ " instance['ProblemsAttempted'] = early_problems.shape[0]\n",
+ " # Percentage of early problems gotten correct eventually\n",
+ " instance['PercCorrectEventually'] = np.mean(early_problems['CorrectEventually'])\n",
+ " # Median attempts made on early problems\n",
+ " instance['MedAttempts'] = np.median(early_problems['Attempts'])\n",
+ " # Max attempts made on early problems\n",
+ " instance['MaxAttempts'] = np.max(early_problems['Attempts'])\n",
+ " # Percentage of problems gotten correct on the first try\n",
+ " instance['PercCorrectFirstTry'] = np.mean(early_problems['Attempts'] == 1)\n",
+ " \n",
+ " instance = instance.drop('SubjectID')\n",
+ " return instance"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "AssignmentID 494\n",
+ "ProblemID 41\n",
+ "ProblemsAttempted 30\n",
+ "PercCorrectEventually 1\n",
+ "MedAttempts 6.5\n",
+ "MaxAttempts 45\n",
+ "PercCorrectFirstTry 0.166667\n",
+ "Name: 0, dtype: object"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "extract_instance_features(X_train_base.iloc[0], early_train)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def extract_instance_code_features(instance, early_df, code_table):\n",
+ " subject_id = instance[PS2.SubjectID]\n",
+ " problem_id = instance[PS2.ProblemID]\n",
+ " \n",
+ " # Get all attempts for this problem by this subject\n",
+ " attempts = code_table[(code_table[PS2.SubjectID] == subject_id) & \\\n",
+ " (code_table[PS2.ProblemID] == problem_id)]\n",
+ " # Get the code of the last attempt (we could use others but don't here)\n",
+ " encoder = code_problem_encoders[problem_id]\n",
+ " # If for some reason there were no attempts, return 0s\n",
+ " if (attempts.shape[0] == 0):\n",
+ " return encoder.transform([\"\"])\n",
+ " last_attempt = attempts.sort_values('Order')['Code'].iloc[-1]\n",
+ " code_features = encoder.transform([last_attempt])\n",
+ " \n",
+ " return code_features"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " (0, 21)\t0.22116095645566827\n",
+ " (0, 19)\t0.2128142305637896\n",
+ " (0, 18)\t0.8512569222551584\n",
+ " (0, 14)\t0.4256284611275792\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(extract_instance_code_features(X_train_base.iloc[0], early_train, code_table_train))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<5x30 sparse matrix of type '<class 'numpy.float64'>'\n",
+ "\twith 37 stored elements in Compressed Sparse Row format>"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Test how to stack code features across instances\n",
+ "from scipy.sparse import vstack\n",
+ "import functools\n",
+ "\n",
+ "code_features = X_train_base.iloc[:5].apply(\\\n",
+ " lambda instance: extract_instance_code_features(\\\n",
+ " instance, early_train, code_table_train), axis=1)\n",
+ "\n",
+ "vstack(code_features)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def extract_features(X, early_df, code_table, scaler, is_train):\n",
+ " # First extract performance features for each row\n",
+ " features = X.apply(lambda instance: extract_instance_features(instance, early_df), axis=1)\n",
+ " # Then get code features\n",
+ " code_features = X.apply(lambda instance: extract_instance_code_features(\\\n",
+ " instance, early_df, code_table), axis=1)\n",
+ " code_features = vstack(code_features).toarray()\n",
+ " \n",
+ " # Then one-hot encode the problem_id and append it\n",
+ " problem_ids = problem_encoder.transform(features[PS2.ProblemID].values.reshape(-1, 1)).toarray()\n",
+ " # Then get rid of nominal features\n",
+ " features.drop([PS2.AssignmentID, PS2.ProblemID], axis=1, inplace=True)\n",
+ " # Then scale the continuous features, fitting the scaler if this is training\n",
+ " if is_train:\n",
+ " scaler.fit(features)\n",
+ " features = scaler.transform(features)\n",
+ " \n",
+ " # Return continuous and one-hot features together\n",
+ " return np.concatenate([features, code_features, problem_ids], axis=1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.preprocessing import StandardScaler\n",
+ "scaler = StandardScaler()\n",
+ "X_train = extract_features(X_train_base, early_train, code_table_train, scaler, True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4201, 55)\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "array([[ 0.51751812, 0.58371895, 1.76922077, 1.70602676, -0.89569333,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0.42562846,\n",
+ " 0. , 0. , 0. , 0.85125692, 0.21281423,\n",
+ " 0. , 0.22116096, 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 1. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ],\n",
+ " [ 0.51751812, 0.58371895, 1.76922077, 1.70602676, -0.89569333,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0.26622699, 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0.22181069, 0.57514592,\n",
+ " 0.21929281, 0.38891882, 0. , 0.57514592, 0.09585765,\n",
+ " 0. , 0.09900339, 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 1. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ]])"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "print(X_train.shape)\n",
+ "X_train[:2,]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Evaluate the Training Performance of the Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.linear_model import LogisticRegressionCV\n",
+ "from sklearn.neural_network import MLPClassifier\n",
+ "from sklearn.ensemble import RandomForestClassifier\n",
+ "\n",
+ "# We set an alpha value; otherwise the model overfits with 100% accuracy\n",
+ "# If we were being rigorous, we would set this using hyperparameter tuning\n",
+ "model = RandomForestClassifier(ccp_alpha=0.001)\n",
+ "model.fit(X_train, y_train)\n",
+ "train_predictions = model.predict(X_train)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " precision recall f1-score support\n",
+ "\n",
+ " False 0.43 0.89 0.58 518\n",
+ " True 0.98 0.83 0.90 3683\n",
+ "\n",
+ " accuracy 0.84 4201\n",
+ " macro avg 0.70 0.86 0.74 4201\n",
+ "weighted avg 0.91 0.84 0.86 4201\n",
+ "\n",
+ "AUC: 0.86040290513546\n",
+ "Macro F1: 0.7377652933832709\n"
+ ]
+ }
+ ],
+ "source": [
+ "from sklearn.metrics import classification_report\n",
+ "from sklearn.metrics import roc_auc_score\n",
+ "from sklearn.metrics import f1_score\n",
+ "\n",
+ "print(classification_report(train_predictions, y_train))\n",
+ "print('AUC: ' + str(roc_auc_score(train_predictions, y_train)))\n",
+ "print('Macro F1: ' + str(f1_score(train_predictions, y_train, average='macro')))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<sklearn.metrics._plot.roc_curve.RocCurveDisplay at 0x1c53d1fd608>"
+ ]
+ },
+ "execution_count": 24,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXwV5bnA8d+ThIQEAmEXiAFEZCcB4oIrVGzRylLFIhctrtxatL0urVZ71VJbd+vF0iJXW3ABVOpCLV5XBEVZJbKDLAHCIiFAyL4+9485OWTnBHIyOZnn+/nkw5nlzDxzQuY57zLvK6qKMcYY7wpzOwBjjDHuskRgjDEeZ4nAGGM8zhKBMcZ4nCUCY4zxuAi3A6ir9u3ba/fu3d0OwxhjQsqaNWsOq2qH6raFXCLo3r07q1evdjsMY4wJKSKyu6ZtVjVkjDEeZ4nAGGM8zhKBMcZ4nCUCY4zxOEsExhjjcUFLBCLydxE5JCIbatguIjJdRLaLyDoRGRKsWIwxxtQsmCWC2cCoWrZfCfTy/UwB/hbEWIwxxtQgaM8RqOpSEeleyy5jgVfUGQd7uYjEiUhnVT0QrJiMMaGnuKSUwpJSMrILKSgu4WBmASKgCopSqqCqKEDZulJQTqzffyyPmMhwoOx9J95ftoxvX/Udz3c4/3bn3PnERTdr+A/B5/K+nUg8M67ej+vmA2Vdgb3lltN866okAhGZglNqICEhoUGCM8bUTW5hMZl5RQBkZBey90guWQXFAJSWOjfsElVKS5Vdh3NYuesIMZHhpB3No7i0lPAwAaCk1LkRl6hyLLfIteupjYg75+3YqnmTSwTVfZTVzpKjqrOAWQDJyck2k44xp6mguITMvCK2HMhi2Y7DxDSL8N+kC4pL2JmeQ5sWkZSUKiWlyrdpx4htHkFpKZSq+n6c1wcz8ylVJb+o9JRiGd67AxnZhfTq1JLI8DBEhPAwCBMhTITjeUWcc0YsuYUl9OrYklJV2reMoll4GGFSdlMWRJybSpiUvfb96ys9tI5u5k82FbY7b6+wLCL+G1TZvghERYTRvFn4aX/+jY2biSANOLPccjyw36VYjGn0VJX0rAK2fp9FYXEpWfnFHM4uICJMyC4oZn9mPrFREZSqUuK7YasqWQXFvP3NPmIiwwkTZ9+ahMmJ6pCoiDA6tooiIiyM8DBh39E8hnZrS5g4N9vwMOfGOaBLa74/ns9ZHVrQKroZPdq1QHFuvAltY/w34LL9w0WICA8jNiqCsDCXvlqbCtxMBAuBO0VkPnA+kGntA8aLsguKOZiZz/fH8zmeV8TW77PIyi/mzdV7adsiElXYcyQ34OOV3fBFIDxMKJuNNrewhJ8N60ZkeBhZ+cWcc0YskeHCOZ1iSe7e1vft2m7MXhS0RCAi84DhQHsRSQMeAZoBqOpMYBFwFbAdyAVuDlYsxritoLiEb3YfY0d6NnuP5FJcqry+YjfRzcI5Wks9eFZ+MWOTunBej7YcySmkS1xz+ndpTa+OLYmJjCCqWRhtYiKJCBeaR4QTGWGPBpm6C2avoYkn2a7A1GCd3xg3bD+UxdJth/nuUBabD2SRsvcYLaMiqq2OaRYuHM0t4upBnenergX9urQiLroZZ7RuTkLbGCLC7aZuGkbIDUNtTGOy/VAWH286xD+/SWP7oexq90loG8PghDgUuG5oPF3joukQG2XVMKbRsERgTB0UFJewfOcR/nfpTr7cfrjK9sT41txycQ+Gn9OR1jHu9Tc3pi4sERhTg//bcJBN+zMpUaW4VEk/XsDba/dV2CcyIoy/TBzMgK6t6RIX7VKkxpweSwTG81SVQ1kFfL0jgxeX7qR1dATLdx7xb28WLhV63wzo2ooHr+zLsJ7trHrHNAmWCIynzF2xh/SsAnIKi1myNZ3MvCIOHs+vsE+bmGb07hTLvmN5vDv1Is7u2NKlaI1pGJYITJO2eMshlu/KYGd6Dmv3HOVwdmGVfa4Z3JVSVUb06Uh8mxiGdmvjQqTGuMcSgWlSjuYUcu3Mr9idkUtJ6YnRSGIiw8ktLKFjbBSv3no+vc+IdTFKYxoXSwSmSSgpVZZ+l87N/1jlX3flgDPo3DqayRd2o1u7Fi5GZ0zjZonAhKRDWfnMXbGHdWmZ7EjPZnfGiSEYzu7Yko/+61Ibx8aYAFkiMCGltFT5j5eWV+jVIwLtW0aR3K0NvxrZi76dW7kYoTGhxxKBCQmlpcq5f/yEjJwTjb03XJDA737cr0kOC2xMQ7JEYBq1nIJivjuUzbgZy/zrfjyoM0+PH0RMpP33NaY+2F+SaZQysgsY+tgnVdZvnjaK6EgrARhTnywRmEZj1+Ec/vzxNrYcPM62708M4PaL4T0Z2LU1I/p0tGogY4LAEoFxnapy25zVfLrlUIX1N1/UnUdG93cpKmO8wxKBcY2qkl1QzJ8WbfYngafGD2L8kHjr+mlMA7JEYBrcloPHGfX8F1XWf/GbEZzZNsaFiIzxNksEpsFs3J/JrsM53Dl3rX/dDRckkNA2hh/1P8OSgDEusURggio9q4BXvk7lhc+2V1if3K0NC+640J2gjDEVWCIwQZF6OIepc79h4/7jFda/eONQ+nVuZd/+jWlELBGYenEkp5CVuzL4y+Lt7D6cS1a5ydpvv6QHP7+sJ+1aRrkYoTGmJpYIzGl7/pNtPP/JdxXWjUvqwlkdWnLH8J40Cw9zKTJjTCAsEZhTlltYzI+eX8reI3kA/GZUb67o24mEdjFERdiDX8aECksE5pQ8vmgzLy7d6V/+5x0X2sxexoQoSwSmTkpLlSc/3OJPAv99dT8mnnemDQBnTAizv14TkJe/3MWKnRl8tOl7/7rnJyQxbnBXF6MyxtQHSwSmRrszcrj9ldUVBoADSDozjv+5PsmmfzSmibBEYKq1af9xrpp+YhiIqwd15r+v7kenVs1djMoYEwyWCEwVa/cc5Sd//QqADrFRrHpopMsRGWOCyTp4mwr2H8vzJ4GbL+puScAYD7ASgaGwuJTnP9nG35bsQPXE+oev7udeUMaYBhPURCAio4D/AcKBl1T1iUrbE4A5QJxvnwdUdVEwYzIVfb71EDf9Y5V/OTYqgimXnsVdl/dyMSpjTEMKWiIQkXBgBnAFkAasEpGFqrqp3G6/A95U1b+JSD9gEdA9WDEZR0Z2AfNX7eWNVXvZcyQXgPN7tGXOLefZVJDGeFAwSwTnAdtVdSeAiMwHxgLlE4ECrXyvWwP7gxiPp6kq76zdx/xVe1m560iFbQ9d1ZfbLz3LpciMMW4LZiLoCuwtt5wGnF9pn0eBj0TkLqAFUG3LpIhMAaYAJCQk1HugTZ2q0uO3J2rcwgTGJnXlyWsHERlh/QWM8bpgJoLqJp3VSssTgdmq+qyIDANeFZEBqlpa4U2qs4BZAMnJyZWPYWqx5eBxbnhppX954+9/RIso6yNgjDkhmHeENODMcsvxVK36uRUYBaCqX4tIc6A9cCiIcXlCdkExt85exYpy1UCrfzfSkoAxpopg3hVWAb1EpAewD7ge+I9K++wBLgdmi0hfoDmQHsSYPOFgZj4XPP6pf/mJawZyzZB4qwYyxlQraIlAVYtF5E7gQ5yuoX9X1Y0iMg1YraoLgXuB/xWRu3GqjW5SVav6OUXHcguZ9v4m3v5mn3/dzj9dRVhYdbV0xhjjCGo9ge+ZgEWV1j1c7vUm4KJgxuAllz+7hIycQgAmnpfA49cMdDkiY0wosArjJuLGl1f4k8COP11FuJUCjDEBskrjJiAzr4gvvjsMwNJfj7AkYIypE0sEIa6kVEn8/UeAM0hcQrsYlyMyxoQaSwQhbla5eYNtkDhjzKmwRBDi5q3cA8C3j/wQEasSMsbUnSWCEPb0h1vYcyQXEWgd3cztcIwxIcp6DYWgb/ce47oXv6aw2BmJ4y8Th7gckTEmlFkiCDG5hcWMnbHMv7zol5fQr0urWt5hjDG1s0QQQuat3MNv314PQIvIcDZOG+VyRMaYpiCgNgIRiRSRs4MdjKnZN3uO+pNAr44tWf/oj1yOyBjTVJy0RCAiPwaeAyKBHiKSBDyiqj8JdnDGcfcbKbyz1hk/aPrEwYxJ7OJyRMaYpiSQqqFpOBPKLAZQ1RQrHTQMVeXy55awMz0HgHuvOMeSgDGm3gWSCIpU9VilPuo2QmiQpR7OYfgzn/uXlz3wA7rGRbsXkDGmyQokEWwWkZ8CYb65BX4FLA9uWN6WnlVQIQmsfOhyOsY2dy8gY0yTFkhj8Z3AUKAUeBvIx0kGJghyC4s594+f+JdTn/ixJQFjTFAFkgh+pKr3q+pg388DwJXBDsyrZi5xxg4akhDH9j/ax2yMCb5AEsHvqln3UH0HYhx7j+QC8PebziUi3EYAMcYEX41tBCLyI5yJ5buKyHPlNrXCqSYy9aykVHln7T7OaNWcuJhIt8MxxnhEbY3Fh4ANOG0CG8utzwIeCGZQXnToeD7n/cmZcD6nsNjlaIwxXlJjIlDVtcBaEXldVfMbMCZP+s0/1/lfr3xwpIuRGGO8JpDuo11F5I9AP8DffUVVzwlaVB6TW1jM51vTAaeXkDHGNKRAWiNnA/8ABKe30JvA/CDG5DmP/XszACP7dnI5EmOMFwWSCGJU9UMAVd2hqr8DRgQ3LO+4bc4q5q5wZhl7/vokl6MxxnhRIFVDBeKML7FDRH4O7AM6Bjcsb1i85RCfbD4EwD9uPpeWUTYquDGm4QVy57kbaAn8Evgj0Bq4JZhBecETH2xh5pIdADx7XSIjeltuNca446SJQFVX+F5mATcCiEh8MINqyo7lFpI07WP/8qOj+3HtUPs4jTHuqTURiMi5QFfgS1U9LCL9gfuBHwB29zoF097f5H/9xW9GcGbbGBejMcaYWhqLReRx4HVgEvB/IvIQzpwE3wLWdfQUvPp1Km9/40wws/NPV1kSMMY0CrWVCMYCiaqaJyJtgf2+5a0NE1rTciSnkP9+z3lA+5nrEgkLk5O8wxhjGkZt3UfzVTUPQFWPAFssCZyanIJihvzBaRe4ZnBXxlubgDGmEamtRHCWiLztey1A93LLqOo1Jzu4iIwC/gcIB15S1Seq2eenwKM4s559q6r/EXj4oeFnf1/pf/3o2P4uRmKMMVXVlgiurbT8l7ocWETCgRnAFUAasEpEFqrqpnL79AJ+C1ykqkdFpMn1obx19irW7D4KwIbf/8ieFTDGNDq1DTr36Wke+zxgu6ruBBCR+TjtDpvK7XM7MENVj/rOeeg0z9mobDl4nE+3OJc09/bzLQkYYxqlYM580hXYW245zbeuvHOAc0RkmYgs91UlVSEiU0RktYisTk9PD1K49a+sJDDzhqFc2LO9y9EYY0z1gpkIqusWo5WWI4BewHBgIvCSiMRVeZPqLFVNVtXkDh061HugwXAwM5+H3tkAQP8urVyOxhhjahZwIhCRqDoeOw04s9xyPE4X1Mr7vKeqRaq6C9iKkxhCWnpWARc87tSsRUWE2fMCxphG7aSJQETOE5H1wHe+5UQReSGAY68CeolIDxGJBK4HFlba5118I5mKSHucqqKddYi/UXp9xW4Azu3ehq2P2QT0xpjGLZASwXTgaiADQFW/JYBhqFW1GLgT+BDYDLypqhtFZJqIjPHt9iGQISKbcJ5a/rWqZtT9MhqX15Y7ieC5n9qw0saYxi+QbixhqrrbGYnarySQg6vqImBRpXUPl3utwD2+nyYhv6iEw9mF9GjfwqqEjDEhIZBEsFdEzgPU92zAXcC24IYVulL2HgNgeO/QaNQ2xphAqobuwPnGngB8D1zgW2cqyS0s5vpZywG4op9NO2mMCQ2BlAiKVfX6oEcS4lSVfg9/6F8+r3tbF6MxxpjABVIiWCUii0RksojEBj2iELXa9/AYOENMR4QH8xENY4ypPye9W6lqT+AxYCiwXkTeFRErIVRyNKcQgLm3nW9DTBtjQkpAX1tV9StV/SUwBDiOM2GNKWf2V6kAtIpu5m4gxhhTR4E8UNZSRCaJyL+AlUA6cGHQIwshpaXKVzucxx/i20S7HI0xxtRNII3FG4B/AU+p6hdBjick7T6SC8DVgzoTFxPpcjTGGFM3gSSCs1S1NOiRhLB1ac6zA9Zl1BgTimpMBCLyrKreC/xTRCqPGhrQDGVekZHtNBT37WyjjBpjQk9tJYI3fP/WaWYyL+sU29ztEIwxps5qm6GsbKLdvqpaIRmIyJ3A6c5g1mR8dyjL7RCMMeaUBdJ99JZq1t1a34GEsnkrnYnYIiPsITJjTOiprY1gAs4cAj1E5O1ym2KBY8EOLFSUlDrNJ9cOiSc6MtzlaIwxpu5qayNYiTMHQTwwo9z6LGBtMIMKJX94fxMAHWLrOoGbMcY0DrW1EewCdgGfNFw4oeftb9IA+K+RIT/DpjHGo2qrGlqiqpeJyFEqTjovOHPKeH54zaz8Io7nF9PnjFiaN7NqIWNMaKqtaqhsOsr2DRFIKHpztVMaONeGnDbGhLAau7mUe5r4TCBcVUuAYcB/Ai0aILZGraRU/e0Dd19xjsvRGGPMqQukv+O7ONNU9gReAfoCc4MaVSOnqvR88MRUzG1b2PhCxpjQFUgiKFXVIuAa4HlVvQvoGtywGrcM39wDAFv+MMrFSIwx5vQFkgiKReQ64Ebgfd86Tw+6P2vpTgB+P6a/NRIbY0JeoE8Wj8AZhnqniPQA5gU3rMbrQGaePxGMG+zpgpExpok46TDUqrpBRH4JnC0ifYDtqvrH4IfW+BQUlzDs8c8AGJPYhdY2G5kxpgk4aSIQkUuAV4F9OM8QnCEiN6rqsmAH19jM8U1HmdytDdMnDnY3GGOMqSeBTEzzZ+AqVd0EICJ9cRJDcjADa2xKSpWn/m8rAH+7YajL0RhjTP0JpI0gsiwJAKjqZsBz/SV3pmdTXKp0axdj4woZY5qUQEoE34jIizilAIBJeHDQudm+aqH7R/VxNxBjjKlngSSCnwO/BH6D00awFHghmEE1RlsPOpPPXNTTRtwwxjQttSYCERkI9ATeUdWnGiakxmn/sTwS41vTOsZ6ChljmpYa2whE5EGc4SUmAR+LSHUzlXlCRnYB+zPzKSrRk+9sjDEhprbG4knAIFW9DjgXuKOuBxeRUSKyVUS2i8gDtew3XkRURBplT6RH/+W0lV87NN7lSIwxpv7VlggKVDUHQFXTT7JvFSISjjOz2ZVAP2CiiPSrZr9YnDaIFXU5fkNRVf717X4AfmJPEhtjmqDa2gjOKjdXsQA9y89drKrXnOTY5+E8hbwTQETmA2OBTZX2+wPwFHBfXQJvKCl7nemZLzq7nY0yaoxpkmpLBNdWWv5LHY/dFdhbbjkNOL/8DiIyGDhTVd8XkRoTgYhMAaYAJCQk1DGM01P2ENntl5zVoOc1xpiGUtucxZ+e5rGlusP6N4qE4Ty1fNPJDqSqs4BZAMnJyQ3WYjt72S6+3pkBwEVnW7dRY0zTVKd6/zpKw5ndrEw8sL/cciwwAPhcRFKBC4CFjaXBOD2rwN9I/PebkmkWHsyPyhhj3BPMu9sqoJeI9BCRSOB6YGHZRlXNVNX2qtpdVbsDy4Exqro6iDEF7NmPnCqh2y7uwQ/6dHI5GmOMCZ6AE4GI1GmAHVUtBu4EPgQ2A2+q6kYRmSYiY+oWZsPKLyph/iqneeO+H/V2ORpjjAmuQIahPg94GWgNJIhIInCbb8rKWqnqImBRpXUP17Dv8EACbghvrXaSwLCz2tkMZMaYJi+QEsF04GogA0BVv8WZsazJmrF4BwBPXzfI5UiMMSb4AkkEYaq6u9K6kmAE0xgsWJPGweP5tG8ZRXybGLfDMcaYoAtk9NG9vuoh9T0tfBewLbhhueexfzs9haZc2sPlSIwxpmEEUiK4A7gHSAC+x+nmWedxh0JBUUkpx3KL+GlyPFMu7el2OMYY0yACmbz+EE7XzyZv0/7jAETYMwPGGA8JpNfQ/1LuieAyqjolKBG56J43UwC4csAZLkdijDENJ5A2gk/KvW4O/ISKYwg1CUUlpexIzwFsFjJjjLcEUjX0RvllEXkV+DhoEbnky+8OAzAkIY6wsOqGSTLGmKbpVCrDewDd6jsQtxWWlAIwbewAlyMxxpiGFUgbwVFOtBGEAUeAGmcbC3VihQFjjMecbPJ6ARKBfb5VparaJCfu/XxrOgBS7ejZxhjTdNVaNeS76b+jqiW+nyaZBOBESeCcTi3dDcQYYxpYIG0EK0VkSNAjaQTat4yyZwiMMZ5TY9WQiET4hpK+GLhdRHYAOTgzj6mqNqnk8M81aTbSqDHGk2prI1gJDAHGNVAsrmpnE9MbYzyqtkQgAKq6o4FicU1OQTH7M/O5ZkhXt0MxxpgGV1si6CAi99S0UVWfC0I8rjiQmQ9A6+hmLkdijDENr7ZEEA60BC/0p3Q6Qw1OaONyHMYY0/BqSwQHVHVag0XiokXrDwIQbk+TGWM8qLa+kp64Kx7OLuC5j515dn7Qp6PL0RhjTMOrLRFc3mBRuOgvn20H4MGr+hAdad1HjTHeU2MiUNUjDRmIGwqKS/j3+gMA3HShTU1pjPEmTz9Gu/dIHulZBdx7xTlERnj6ozDGeJin736vfJ0KwIVnt3M1DmOMcZOnE8HBzHx6dmjB0G5t3Q7FGGNc4+lEANDMBpkzxnicZ++CJaXKR5u+dzsMY4xxnWcTwfKdGQAcySl0ORJjjHGXZxPBx77SwMuTz3U5EmOMcZdnE0HZw2MDurZyORJjjHFXUBOBiIwSka0isl1Eqkx4LyL3iMgmEVknIp+KSLdgxlNZZHgYYuMLGWM8LmiJQETCgRnAlUA/YKKI9Ku021ogWVUHAQuAp4IVjzHGmOoFs0RwHrBdVXeqaiEwHxhbfgdVXayqub7F5UB8EOOpQLWhzmSMMY1bMBNBV2BvueU037qa3Ap8UN0GEZkiIqtFZHV6enq9BLdxfyZRNqyEMcYENRFUV/le7fdwEbkBSAaerm67qs5S1WRVTe7QoUO9BJeRXUi/LtZQbIwxtU1Mc7rSgDPLLccD+yvvJCIjgYeAy1S1IIjxVBHb3KamNMaYYJYIVgG9RKSHiEQC1wMLy+8gIoOBF4ExqnooiLEYY4ypQdASgaoWA3cCHwKbgTdVdaOITBORMb7dnsaZF/ktEUkRkYU1HM4YY0yQBLNqCFVdBCyqtO7hcq9HBvP8tdmfmUeXuGi3Tm+MMY2GJ7vNZOUXcSy3iMw8G2fIGGM8mQjyi0oBGN7bJqs3xhhPJoJv9hwFIL6NVQ0ZY4wnE0F6ltNLNenMOJcjMcYY93kyESzZ5jydXDYCqTHGeJknE0F0s3BaNY+gY2xzt0MxxhjXeTIRALRrGeV2CMYY0yh4MhF8uf0wpTb8qDHGAEF+oKyxyissoaTUEoExxoBHSwSREWH8ZHBtI2IbY4x3eDIRZOYVuR2CMcY0Gp5LBNsPZQOQXVDsciTGGNM4eC4R/HLeWgBG2PASxhgDeCwR5BWWsOnAcQAuPae9y9EYY0zj4KlEUFzqDDb3wJV9bHYyY4zx8VQiKGskbhnlyV6zxhhTLU8lgi0HsgDo2znW5UiMMabx8FQiKCh2qoasWsgYY07wVCIwxhhTlSUCY4zxOE8lgtSMHADaxES6HIkxxjQenuo+szsjhzNaNadDrA1B3RCKiopIS0sjPz/f7VCM8YzmzZsTHx9Ps2aBt4V6KhEUlygibkfhHWlpacTGxtK9e3fEPnhjgk5VycjIIC0tjR49egT8Pk9VDaXsPUafM6zraEPJz8+nXbt2lgSMaSAiQrt27epcCvdUIsgpLKZTK5uesiFZEjCmYZ3K35xnEoGqciSn0O0wjDGm0fFMIth1OIciayPwnPDwcJKSkhgwYACjR4/m2LFj9XLc1NRUBgwYUC/Huummm+jRowdJSUkkJSUxffr0ejludT7//HO++uqrCuteeeUVBgwYQP/+/enXrx/PPPOMP64FCxbUy3n379/P+PHj/csTJ05k0KBB/PnPf+bhhx/mk08+Oa3jv/vuu0ybNq3CusTERCZOnFhh3fDhw1m9erV/ufLvceXKlVx66aX07t2bPn36cNttt5Gbm3tase3atYvzzz+fXr16MWHCBAoLq34hLSoqYvLkyQwcOJC+ffvy+OOPA7B161b//4ukpCRatWrF888/D8B9993HZ599dlqx+alqSP0MHTpUT8XmA5na7f739f1v95/S+03dbdq0ye0QtEWLFv7XP/vZz/Sxxx6rl+Pu2rVL+/fvXy/Hmjx5sr711lun9N7i4uI67f/II4/o008/7V9etGiRDh48WPft26eqqnl5eTpr1qzTjqs2Bw4c0ISEhFN+f1FRUZV1w4YN0/T0dP/ypk2bdMCAAdqlSxfNzs72r7/ssst01apV/uXyv8eDBw9qQkKCfvXVV6qqWlpaqm+99ZYePHjwlGNVVb3uuut03rx5qqr6n//5n/rXv/61yj6vv/66TpgwQVVVc3JytFu3brpr164K+xQXF2unTp00NTVVVVVTU1P1iiuuqPac1f3tAau1hvuqp3oNAYRZicAVv//XRjbtP16vx+zXpRWPjO4f8P7Dhg1j3bp1AGRnZzN27FiOHj1KUVERjz32GGPHjiU1NZUrr7ySiy++mK+++oquXbvy3nvvER0dzZo1a7jllluIiYnh4osv9h83Pz+fO+64g9WrVxMREcFzzz3HiBEjmD17Nu+++y4lJSVs2LCBe++9l8LCQl599VWioqJYtGgRbdu2rTHeefPm8ac//QlV5cc//jFPPvkkAC1btuSee+7hww8/5NlnnyU6Opp77rmH7Oxs2rdvz+zZs+ncuTPTp09n5syZRERE0K9fP5544glmzpxJeHg4r732Gi+88AKPP/44zzzzDF26dAGcroe33357lVimTZvGv/71L/Ly8rjwwgt58cUXEZEq55g/fz5LlizhV7/6FeDUVy9dupSMjAyuvvpqNmzYwA9/+EMOHTpEUlISL7zwAi+//DJXX30148ePZ82aNdVey/Dhw7nwwgtZtmwZY8aM4d577/XHtm3bNqKiomjf/sTQ8nPnzuXGG29k8+bNLFy4sErJoDozZsxg8uTJDBs2zB97+VLMqVBVPg/OjLMAABCwSURBVPvsM+bOnQvA5MmTefTRR7njjjsq7Cci5OTkUFxcTF5eHpGRkbRq1arCPp9++ik9e/akW7duAHTr1o2MjAwOHjzIGWeccVpxeqZqyHhbSUkJn376KWPGjAGcG94777zDN998w+LFi7n33ntxvjTBd999x9SpU9m4cSNxcXH885//BODmm29m+vTpfP311xWOPWPGDADWr1/PvHnzmDx5sr/XxoYNG5g7dy4rV67koYceIiYmhrVr1zJs2DBeeeUV/zF+/etf+4v/69evZ//+/dx///189tlnpKSksGrVKt59910AcnJyGDBgACtWrOD888/nrrvuYsGCBf5E9dBDDwHwxBNPsHbtWtatW8fMmTPp3r07P//5z7n77rtJSUnhkksuYcOGDQwdOvSkn9+dd97JqlWr2LBhA3l5ebz//vvVngPgmWeeYcaMGaSkpPDFF18QHR1d4VgLFy6kZ8+e/hjKFBUV1XgtAMeOHWPJkiUVkgDAsmXLGDJkSIV1b7zxBhMmTGDixInMmzfvpNcHBPxZVK6uKf9TueoxIyODuLg4IiKc79zx8fHs27evyjHHjx9PixYt6Ny5MwkJCdx3331VviTMnz+/SkIbMmQIy5YtC+j6auO5EoFxR12+udenvLw8kpKSSE1NZejQoVxxxRWA803twQcfZOnSpYSFhbFv3z6+//57AH99PcDQoUNJTU0lMzOTY8eOcdlllwFw44038sEHHwDw5ZdfctdddwHQp08funXrxrZt2wAYMWIEsbGxxMbG0rp1a0aPHg3AwIED/aUTgKeffrrCt8/33nuP4cOH06FDBwAmTZrE0qVLGTduHOHh4Vx77bWAc1PasGGD/7pKSkro3LkzAIMGDWLSpEmMGzeOcePGndbnuHjxYp566ilyc3M5cuQI/fv3Z/To0dWe46KLLuKee+5h0qRJXHPNNcTHxwd0jtquBWDChAnVvu/AgQP+zwlg1apVdOjQgW7duhEfH88tt9zC0aNHadOmTbU9auray6Z3796kpKQEtG/Zl4uTnW/lypWEh4ezf/9+jh49yiWXXMLIkSM566yzACgsLGThwoX+toMyHTt2ZP/+/XWKvzpBLRGIyCgR2Soi20XkgWq2R4nIG77tK0Ske7BiKSpW3zmDdQbTGEVHR5OSksLu3bspLCz0f3t//fXXSU9PZ82aNaSkpNCpUyf/t/ioqBNPnoeHh1NcXIyq1njDqO6PvUz5Y4WFhfmXw8LCKC6ued7s2o7ZvHlzwsPD/fv179+flJQUUlJSWL9+PR999BEA//73v5k6dSpr1qxh6NCh1Z6vf//+rFmzpsZzgVP19Ytf/IIFCxawfv16br/9dv9nVd05HnjgAV566SXy8vK44IIL2LJlS63HL3/NNV0LQIsWLap9X3R0dIV+8/PmzWPLli10796dnj17cvz4cX+prl27dhw9etS/75EjR/xVSoF8FlC3EkH79u05duyY/7NPS0vzV8OVN3fuXEaNGkWzZs3o2LEjF110UYVG7Q8++IAhQ4bQqVOnCu/Lz8+vUuI6FUFLBCISDswArgT6ARNFpF+l3W4Fjqrq2cCfgSeDFc83e5xffr/OrYN1CtOItW7dmunTp/PMM89QVFREZmYmHTt2pFmzZixevJjdu3fX+v64uDhat27Nl19+CTiJpMyll17qX962bRt79uyhd+/epxXv+eefz5IlSzh8+DAlJSXMmzfPXxopr3fv3qSnp/urq4qKiti4cSOlpaXs3buXESNG8NRTT3Hs2DGys7OJjY0lKyvL//7f/va3/OY3v+HgwYMAFBQUVOm1VHaTbd++PdnZ2f6eRDWdY8eOHQwcOJD777+f5OTkgBNBTddyMn379mX79u3+mN566y3WrVtHamoqqampvPfee/7qoeHDh/Paa6/5E+2cOXMYMWIE4FR/zZkzhxUrVviP/dprr/k/m/JxliWryj9xcXEV9hURRowY4f/M5syZw9ixY6tcQ0JCAp999hmqSk5ODsuXL6dPnz7+7fPmzau2nWPbtm310nstmCWC84DtqrpTVQuB+UDlT2AsMMf3egFwuQTpCaT8ohIA2sfagHNeNXjwYBITE5k/fz6TJk1i9erVJCcn8/rrr1f4o6vJP/7xD6ZOncqwYcMqfAv7xS9+QUlJCQMHDmTChAnMnj27QkngVHTu3JnHH3+cESNGkJiYyJAhQ6q9gURGRrJgwQLuv/9+EhMTSUpK4quvvqKkpIQbbriBgQMHMnjwYO6++27i4uIYPXo077zzDklJSXzxxRdcddVVTJ06lZEjR9K/f/9qSw5xcXHcfvvtDBw4kHHjxnHuuecC1HiO559/ngEDBpCYmEh0dDRXXnllQNdc07WczKWXXsratWtRVZYuXUrXrl3p2rVrhe2bNm3iwIEDTJkyhdjYWBITE0lMTCQ7O5v77rsPgE6dOjF//nzuu+8+evfuTd++ffniiy+qNNrW1ZNPPslzzz3H2WefTUZGBrfeeivgtJU8/PDDAEydOpXs7GwGDBjAueeey80338ygQYMAyM3N5eOPP+aaa66pcNyioiK2b99OcnLyacUHILUVQU/rwCLjgVGqeptv+UbgfFW9s9w+G3z7pPmWd/j2OVzpWFOAKQAJCQlDT/btrTofbTzIuyn7eO6nSTRvFn6ql2XqYPPmzfTt29ftMIwH/OpXv2L06NGMHDnS7VAaTFlnhz/84Q9VtlX3tycia1S12qwRzBJBdd/sK2edQPZBVWeparKqJpdvFKqLH/Y/g79OGmpJwJgm6MEHHzztB79CTXFxcZUeVKcqmL2G0oAzyy3HA5Wbt8v2SRORCKA1cCSIMRljmqBOnTr5uwZ7xXXXXVdvxwpmiWAV0EtEeohIJHA9sLDSPguByb7X44HPNFh1VcYV9us0pmGdyt9c0BKBqhYDdwIfApuBN1V1o4hME5Gy1P0y0E5EtgP3AFW6mJrQ1bx5czIyMiwZGNNA1DcfQfPmdRtlOWiNxcGSnJys5fvXmsbLZigzpuHVNENZbY3F9mSxCZpmzZrVaZYkY4w7bKwhY4zxOEsExhjjcZYIjDHG40KusVhE0oG6P1rsaA8cPuleTYtdszfYNXvD6VxzN1Wt9onckEsEp0NEVtfUat5U2TV7g12zNwTrmq1qyBhjPM4SgTHGeJzXEsEstwNwgV2zN9g1e0NQrtlTbQTGGGOq8lqJwBhjTCWWCIwxxuOaZCIQkVEislVEtotIlRFNRSRKRN7wbV8hIt0bPsr6FcA13yMim0RknYh8KiLd3IizPp3smsvtN15EVERCvqthINcsIj/1/a43isjcho6xvgXwfztBRBaLyFrf/++r3IizvojI30XkkG8Gx+q2i4hM930e60RkyGmfVFWb1A8QDuwAzgIigW+BfpX2+QUw0/f6euANt+NugGseAcT4Xt/hhWv27RcLLAWWA8lux90Av+dewFqgjW+5o9txN8A1zwLu8L3uB6S6HfdpXvOlwBBgQw3brwI+wJnh8QJgxemesymWCM4DtqvqTlUtBOYDlWf9HgvM8b1eAFwuItVNmxkqTnrNqrpYVcvm8luOM2NcKAvk9wzwB+ApoCmMhR3INd8OzFDVowCqeqiBY6xvgVyzAmUzzLem6kyIIUVVl1L7TI1jgVfUsRyIE5HOp3POppgIugJ7yy2n+dZVu486E+hkAu0aJLrgCOSay7sV5xtFKDvpNYvIYOBMVX2/IQMLokB+z+cA54jIMhFZLiKjGiy64Ajkmh8FbhCRNGARcFfDhOaauv69n1RTnI+gum/2lfvIBrJPKAn4ekTkBiAZuCyoEQVfrdcsImHAn4GbGiqgBhDI7zkCp3poOE6p7wsRGaCqx4IcW7AEcs0Tgdmq+qyIDANe9V1zafDDc0W937+aYokgDTiz3HI8VYuK/n1EJAKnOFlbUayxC+SaEZGRwEPAGFUtaKDYguVk1xwLDAA+F5FUnLrUhSHeYBzo/+33VLVIVXcBW3ESQ6gK5JpvBd4EUNWvgeY4g7M1VQH9vddFU0wEq4BeItJDRCJxGoMXVtpnITDZ93o88Jn6WmFC1Emv2VdN8iJOEgj1emM4yTWraqaqtlfV7qraHaddZIyqhvI8p4H8334Xp2MAItIep6poZ4NGWb8CueY9wOUAItIXJxGkN2iUDWsh8DNf76ELgExVPXA6B2xyVUOqWiwidwIf4vQ4+LuqbhSRacBqVV0IvIxTfNyOUxK43r2IT1+A1/w00BJ4y9cuvkdVx7gW9GkK8JqblACv+UPghyKyCSgBfq2qGe5FfXoCvOZ7gf8VkbtxqkhuCuUvdiIyD6dqr72v3eMRoBmAqs7EaQe5CtgO5AI3n/Y5Q/jzMsYYUw+aYtWQMcaYOrBEYIwxHmeJwBhjPM4SgTHGeJwlAmOM8ThLBKbREZESEUkp99O9ln271zRKYx3P+blvhMtvfcMz9D6FY/xcRH7me32TiHQpt+0lEelXz3GuEpGkAN7zXyISc7rnNk2XJQLTGOWpalK5n9QGOu8kVU3EGZDw6bq+WVVnquorvsWbgC7ltt2mqpvqJcoTcf6VwOL8L8ASgamRJQITEnzf/L8QkW98PxdWs09/EVnpK0WsE5FevvU3lFv/ooiEn+R0S4Gzfe+93DfO/XrfOPFRvvVPyIn5HZ7xrXtURO4TkfE44zm97jtntO+bfLKI3CEiT5WL+SYReeEU4/yacoONicjfRGS1OPMQ/N637pc4CWmxiCz2rfuhiHzt+xzfEpGWJzmPaeIsEZjGKLpctdA7vnWHgCtUdQgwAZhezft+DvyPqibh3IjTfEMOTAAu8q0vASad5PyjgfUi0hyYDUxQ1YE4T+LfISJtgZ8A/VV1EPBY+Ter6gJgNc439yRVzSu3eQFwTbnlCcAbpxjnKJwhJco8pKrJwCDgMhEZpKrTccahGaGqI3zDTvwOGOn7LFcD95zkPKaJa3JDTJgmIc93MyyvGfAXX514Cc4YOpV9DTwkIvHA26r6nYhcDgwFVvmG1ojGSSrVeV1E8oBUnKGMewO7VHWbb/scYCrwF5z5DV4SkX8DAQ9zrarpIrLTN0bMd75zLPMdty5xtsAZcqH87FQ/FZEpOH/XnXEmaVlX6b0X+NYv850nEudzMx5micCEiruB74FEnJJslYlmVHWuiKwAfgx8KCK34QzZO0dVfxvAOSaVH5RORKqdo8I3/s15OAOdXQ/cCfygDtfyBvBTYAvwjqqqOHflgOPEmanrCWAGcI2I9ADuA85V1aMiMhtn8LXKBPhYVSfWIV7TxFnVkAkVrYEDvjHmb8T5NlyBiJwF7PRVhyzEqSL5FBgvIh19+7SVwOdr3gJ0F5Gzfcs3Akt8deqtVXURTkNsdT13snCGwq7O28A4nHH03/Ctq1OcqlqEU8Vzga9aqRWQA2SKSCfgyhpiWQ5cVHZNIhIjItWVroyHWCIwoeKvwGQRWY5TLZRTzT4TgA0ikgL0wZnObxPODfMjEVkHfIxTbXJSqpqPM7LjWyKyHigFZuLcVN/3HW8JTmmlstnAzLLG4krHPQpsArqp6krfujrH6Wt7eBa4T1W/xZmreCPwd5zqpjKzgA9EZLGqpuP0aJrnO89ynM/KeJiNPmqMMR5nJQJjjPE4SwTGGONxlgiMMcbjLBEYY4zHWSIwxhiPs0RgjDEeZ4nAGGM87v8B/9fPy2Xwks4AAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "from sklearn.metrics import plot_roc_curve\n",
+ "\n",
+ "plot_roc_curve(model, X_train, y_train)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[ 9 28 31 6 34 17 11 7 45 29 42 53 14 36 27 39 41 40 48 54 46 35 49 38\n",
+ " 50 43 22 30 52 25 51 13 47 44 10 32 5 37 15 12 8 21 33 0 23 16 18 19\n",
+ " 20 26 3 24 2 1 4]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "array([2.20181309e-02, 1.30740725e-01, 1.25642876e-01, 7.78558802e-02,\n",
+ " 1.71925190e-01, 8.73186425e-03, 0.00000000e+00, 6.09759997e-05,\n",
+ " 1.57659859e-02, 0.00000000e+00, 7.27993663e-03, 0.00000000e+00,\n",
+ " 1.51091871e-02, 3.42760977e-03, 9.11457338e-04, 1.22381474e-02,\n",
+ " 3.29994376e-02, 0.00000000e+00, 3.32649011e-02, 4.05234308e-02,\n",
+ " 4.69469226e-02, 1.62030641e-02, 2.25987032e-03, 2.81709411e-02,\n",
+ " 8.25541671e-02, 2.90669657e-03, 4.85810729e-02, 1.03981661e-03,\n",
+ " 0.00000000e+00, 4.79588211e-04, 2.38210644e-03, 0.00000000e+00,\n",
+ " 7.65973842e-03, 1.93997924e-02, 0.00000000e+00, 1.35906502e-03,\n",
+ " 9.82319379e-04, 1.03365796e-02, 1.50237139e-03, 1.17278967e-03,\n",
+ " 1.19422740e-03, 1.19220858e-03, 7.76112145e-04, 2.21994738e-03,\n",
+ " 4.11228179e-03, 4.42651252e-04, 1.32946917e-03, 3.96809141e-03,\n",
+ " 1.30278686e-03, 1.37757687e-03, 2.12040369e-03, 2.94789037e-03,\n",
+ " 2.45866110e-03, 8.13472736e-04, 1.31158028e-03])"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Look at which features were important\n",
+ "print(np.argsort(model.feature_importances_))\n",
+ "model.feature_importances_"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Index(['AssignmentID', 'ProblemID', 'ProblemsAttempted',\n",
+ " 'PercCorrectEventually', 'MedAttempts', 'MaxAttempts',\n",
+ " 'PercCorrectFirstTry'],\n",
+ " dtype='object')"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# The first 7 featurs are from attempts.\n",
+ "# The top-4 most important are from these.\n",
+ "extract_instance_features(X_train_base.iloc[0], early_train).index"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "['10',\n",
+ " '21',\n",
+ " 'big',\n",
+ " 'boolean',\n",
+ " 'charat',\n",
+ " 'count',\n",
+ " 'day',\n",
+ " 'else',\n",
+ " 'end',\n",
+ " 'equals',\n",
+ " 'false',\n",
+ " 'for',\n",
+ " 'goal',\n",
+ " 'if',\n",
+ " 'int',\n",
+ " 'length',\n",
+ " 'new',\n",
+ " 'num',\n",
+ " 'nums',\n",
+ " 'public',\n",
+ " 'result',\n",
+ " 'return',\n",
+ " 'small',\n",
+ " 'speed',\n",
+ " 'str',\n",
+ " 'string',\n",
+ " 'substring',\n",
+ " 'sum',\n",
+ " 'true',\n",
+ " 'word']"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# The next 30 features come from the vocbulary, and the next 20 are the test problem ID\n",
+ "sorted(top_vocab)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "num\n",
+ "public\n",
+ "goal\n",
+ "if\n",
+ "for\n"
+ ]
+ }
+ ],
+ "source": [
+ "# The 5th highest importance feature is #24\n",
+ "# The variable name num\n",
+ "print(sorted(top_vocab)[24 - 7])\n",
+ "# public, perhaps used if declaring a helper method\n",
+ "print(sorted(top_vocab)[26 - 7])\n",
+ "# The variable name goal\n",
+ "print(sorted(top_vocab)[19 - 7])\n",
+ "# if\n",
+ "print(sorted(top_vocab)[20 - 7])\n",
+ "# for\n",
+ "print(sorted(top_vocab)[18 - 7])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Evaluate the CV Performance of the Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Accuracy: 0.7621963648717514\n",
+ "AUC: 0.7716510786626823\n",
+ "Macro F1: 0.6024647798356945\n"
+ ]
+ }
+ ],
+ "source": [
+ "from sklearn.model_selection import cross_validate\n",
+ "\n",
+ "model = RandomForestClassifier(ccp_alpha=0.001)\n",
+ "cv_results = cross_validate(model, X_train, y_train, cv=5, scoring=['accuracy', 'f1_macro', 'roc_auc'])\n",
+ "print(f'Accuracy: {np.mean(cv_results[\"test_accuracy\"])}')\n",
+ "print(f'AUC: {np.mean(cv_results[\"test_roc_auc\"])}')\n",
+ "print(f'Macro F1: {np.mean(cv_results[\"test_f1_macro\"])}')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Predict on the test data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "early_test = pd.read_csv(os.path.join(TEST_PATH, 'early.csv'))\n",
+ "late_test = pd.read_csv(os.path.join(TEST_PATH, 'late.csv'))\n",
+ "\n",
+ "test_ps2 = ProgSnap2Dataset(os.path.join(TEST_PATH, 'Data'))\n",
+ "code_table_test = get_code_table(test_ps2)\n",
+ "\n",
+ "X_test = extract_features(late_test, early_test, code_table_test, scaler, False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(1511, 55)"
+ ]
+ },
+ "execution_count": 31,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "X_test.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = RandomForestClassifier(ccp_alpha=0.001)\n",
+ "model.fit(X_train, y_train)\n",
+ "predictions = model.predict_proba(X_test)[:,1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([0.54347246, 0.53288555, 0.55693535, ..., 0.49766704, 0.50718469,\n",
+ " 0.50822475])"
+ ]
+ },
+ "execution_count": 33,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "predictions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Label</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>036ad3e516c5bf3a4b3be35b137bcbb8</td>\n",
+ " <td>494.0</td>\n",
+ " <td>41</td>\n",
+ " <td>0.543472</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>036ad3e516c5bf3a4b3be35b137bcbb8</td>\n",
+ " <td>494.0</td>\n",
+ " <td>43</td>\n",
+ " <td>0.532886</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>036ad3e516c5bf3a4b3be35b137bcbb8</td>\n",
+ " <td>494.0</td>\n",
+ " <td>44</td>\n",
+ " <td>0.556935</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>036ad3e516c5bf3a4b3be35b137bcbb8</td>\n",
+ " <td>494.0</td>\n",
+ " <td>46</td>\n",
+ " <td>0.530884</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>036ad3e516c5bf3a4b3be35b137bcbb8</td>\n",
+ " <td>494.0</td>\n",
+ " <td>49</td>\n",
+ " <td>0.550823</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1506</th>\n",
+ " <td>fc5f86251458722c799d1830fa0c2c1f</td>\n",
+ " <td>494.0</td>\n",
+ " <td>67</td>\n",
+ " <td>0.504612</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1507</th>\n",
+ " <td>fc5f86251458722c799d1830fa0c2c1f</td>\n",
+ " <td>494.0</td>\n",
+ " <td>104</td>\n",
+ " <td>0.501157</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1508</th>\n",
+ " <td>fc5f86251458722c799d1830fa0c2c1f</td>\n",
+ " <td>494.0</td>\n",
+ " <td>106</td>\n",
+ " <td>0.497667</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1509</th>\n",
+ " <td>fc5f86251458722c799d1830fa0c2c1f</td>\n",
+ " <td>494.0</td>\n",
+ " <td>107</td>\n",
+ " <td>0.507185</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1510</th>\n",
+ " <td>fc5f86251458722c799d1830fa0c2c1f</td>\n",
+ " <td>494.0</td>\n",
+ " <td>108</td>\n",
+ " <td>0.508225</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>1511 rows × 4 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Label\n",
+ "0 036ad3e516c5bf3a4b3be35b137bcbb8 494.0 41 0.543472\n",
+ "1 036ad3e516c5bf3a4b3be35b137bcbb8 494.0 43 0.532886\n",
+ "2 036ad3e516c5bf3a4b3be35b137bcbb8 494.0 44 0.556935\n",
+ "3 036ad3e516c5bf3a4b3be35b137bcbb8 494.0 46 0.530884\n",
+ "4 036ad3e516c5bf3a4b3be35b137bcbb8 494.0 49 0.550823\n",
+ "... ... ... ... ...\n",
+ "1506 fc5f86251458722c799d1830fa0c2c1f 494.0 67 0.504612\n",
+ "1507 fc5f86251458722c799d1830fa0c2c1f 494.0 104 0.501157\n",
+ "1508 fc5f86251458722c799d1830fa0c2c1f 494.0 106 0.497667\n",
+ "1509 fc5f86251458722c799d1830fa0c2c1f 494.0 107 0.507185\n",
+ "1510 fc5f86251458722c799d1830fa0c2c1f 494.0 108 0.508225\n",
+ "\n",
+ "[1511 rows x 4 columns]"
+ ]
+ },
+ "execution_count": 34,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "predictions_df = late_test.copy()\n",
+ "predictions_df['Label'] = predictions\n",
+ "predictions_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# We don't have the test labels - you have to submit to evaluate it"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# We use res/predictions.csv, since that's where the scoring rogram expects it\n",
+ "# but you can change this directory\n",
+ "path = os.path.join('data', 'Prediction', semester, 'code_RF_task1', 'res')\n",
+ "os.makedirs(path, exist_ok=True)\n",
+ "predictions_df.to_csv(os.path.join(path, 'predictions.csv'), index=False)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/compare_semesters.ipynb b/compare_semesters.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..a92d1e62c16461b44320100832cf91caf85c4ae9
--- /dev/null
+++ b/compare_semesters.ipynb
@@ -0,0 +1,565 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "from ProgSnap2 import ProgSnap2Dataset\n",
+ "from ProgSnap2 import PS2\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "import numpy as np\n",
+ "import os\n",
+ "from os import path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "PATH = \"data/CodeWorkout/\"\n",
+ "\n",
+ "s19_ps2 = ProgSnap2Dataset(PATH + 'S19')\n",
+ "f19_ps2 = ProgSnap2Dataset(PATH + 'F19')\n",
+ "\n",
+ "s19 = s19_ps2.get_main_table()\n",
+ "f19 = f19_ps2.get_main_table()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "s19_problems = set(s19[PS2.ProblemID].unique())\n",
+ "f19_problems = set(f19[PS2.ProblemID].unique())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([439., 487., 492., 494., 502.])"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sort(s19[PS2.AssignmentID].unique())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([597, 600, 609, 615, 622, 631], dtype=int64)"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.sort(f19[PS2.AssignmentID].unique())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "48"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# 48 problems the same\n",
+ "len(s19_problems.intersection(f19_problems))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{45, 48}"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Problems 45 and 48 were dropped (from assignment 5)\n",
+ "dropped_problems = s19_problems.difference(f19_problems)\n",
+ "dropped_problems"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{30, 171, 241, 242, 243, 244, 245, 246, 254, 255, 736, 737}"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# 12 problems were added\n",
+ "added_problems = f19_problems.difference(s19_problems)\n",
+ "added_problems"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>73845</th>\n",
+ " <td>622</td>\n",
+ " <td>241</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>73862</th>\n",
+ " <td>622</td>\n",
+ " <td>171</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>73880</th>\n",
+ " <td>622</td>\n",
+ " <td>30</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>73900</th>\n",
+ " <td>622</td>\n",
+ " <td>244</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>73904</th>\n",
+ " <td>622</td>\n",
+ " <td>245</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>73912</th>\n",
+ " <td>622</td>\n",
+ " <td>246</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>73922</th>\n",
+ " <td>622</td>\n",
+ " <td>254</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>73927</th>\n",
+ " <td>622</td>\n",
+ " <td>255</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>73973</th>\n",
+ " <td>622</td>\n",
+ " <td>242</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>74023</th>\n",
+ " <td>622</td>\n",
+ " <td>243</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>92628</th>\n",
+ " <td>631</td>\n",
+ " <td>736</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>92639</th>\n",
+ " <td>631</td>\n",
+ " <td>737</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " AssignmentID ProblemID\n",
+ "73845 622 241\n",
+ "73862 622 171\n",
+ "73880 622 30\n",
+ "73900 622 244\n",
+ "73904 622 245\n",
+ "73912 622 246\n",
+ "73922 622 254\n",
+ "73927 622 255\n",
+ "73973 622 242\n",
+ "74023 622 243\n",
+ "92628 631 736\n",
+ "92639 631 737"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# 10 Were added to a new assignment and 2 replace the old problems in assignment 5\n",
+ "f19[f19[PS2.ProblemID].isin(added_problems)][[PS2.AssignmentID, PS2.ProblemID]].drop_duplicates()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def time_stats(df):\n",
+ " df = df.copy()\n",
+ " df['TimeInt'] = pd.to_datetime(df[PS2.ServerTimestamp]).apply(lambda x: x.value)\n",
+ " med_time = df.groupby([PS2.AssignmentID, PS2.ProblemID])['TimeInt'].apply(lambda x: np.median(x))\n",
+ " # df = df.merge(med_time.to_frame('MedTime'), on=[PS2.AssignmentID, PS2.ProblemID])\n",
+ " return med_time.reset_index().sort_values('TimeInt')\n",
+ "\n",
+ "s19_times = time_stats(s19)\n",
+ "f19_times = time_stats(f19)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<matplotlib.collections.PathCollection at 0x2102ad0f8c8>"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEFCAYAAAD69rxNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAbrElEQVR4nO3de5BV5Znv8e/TV2hoBKRBBBQFb8Hy2nahTDQaVPQk3kaMg7GYioZoGY9nzliZzKTGTCZORU0ZHT1jFCMKUxHi8XgfxYiJmjMi2ioIKIZroMWBVkFEoOnLM3+sTWx7r01371577bV2/z5VXd39rr33+7w2/XP1u9d6X3N3REQkfcqKXYCIiORHAS4iklIKcBGRlFKAi4iklAJcRCSlKuLsbMSIET5+/Pg4uxQRSb0333zzI3ev69oea4CPHz+exsbGOLsUEUk9M/tTWLumUEREUqpHZ+BmtgH4DGgH2ty93syGA78BxgMbgMvcfVthyhQRka56cwZ+pruf4O71me9/CLzo7kcAL2a+FxGRmPRlCuVCYG7m67nARX0vR0REeqqnAe7Ab83sTTOblWkb5e4fAmQ+jwx7opnNMrNGM2tsbm7ue8UiIinS3AxLl8LOndG/dk8DfIq7nwScB1xnZqf3tAN3n+3u9e5eX1eXdRWMiEhJ2rMHZsyAQw6BM86AkSPhppsgyvUDexTg7r4583kr8DjQAGwxs9EAmc9boytLRCTdbrgBnngiCPIdO2D3brj9dnjwwej66DbAzWyQmdXu+xo4B1gBPAXMzDxsJvBkdGWJiKRXSwvMmxeEdme7dsGtt0bXT08uIxwFPG5m+x7/sLsvNLM3gEfM7CpgIzA9urJERNLrs8+goyP8WJRvBXYb4O6+Djg+pP1j4OvRlSIiUhoOPBCGDIGPPvpyuxlMmRJdP7oTU0QkYqtWBfPeXZnBz34WXT8KcBGRiN19N7S3Z7dXVwfz41FRgIuIRGz9+vAAr6yEDz6Irh8FuIhIxKZOhaqq7PaWFqivz27PlwJcRCRi9fXQ2prdPmIEHHxwdP0owEVEIjZ7dnj7xx/D2rXR9aMAFxGJ2Nq14bfMV1dDU1N0/SjARUQidsYZwRuWXe3ZA8ceG10/CnARkYhNnRo+Bz5mTHCTT1QU4CIiEZszJ7hpp6sPPoCNG6PrRwEuIhKx1avD58AHDIBNm6LrRwEuIhKxKVOgImSlqZYWOOaY6PpRgIuIROzss6GtLbt9/HgYPjy6fhTgIiIRmzMnvH39+miXk1WAi4hE7P33w9urqzUHLiKSaA0NUBaSrnv3wsSJ0fWjABcRiVhDQ/iOPA0NwUYPUVGAi4hE7Je/DG9fujR8mdl8KcBFRCK2fn14+549sHNndP0owEVEIrJyZbAOSq6QHjIEamuj668nu9KLiEg3PvwQTjstfC9MgJoauPnm8Dc386UzcBGRCNxzT+79LseNg/vug+99L9o+dQYuIhKBt98OD/AhQ+DOO+GSS6LvU2fgIiIROPnk8H0wW1ujXf+kMwW4iEgELr44uFGnq8GDFeAiIon2yCPhu/Ds2AFr1hSmTwW4iEgEGhvDd+Gprob33itMnwpwEZEInHACVFVm3z/f2uoceWRh+lSAi4hE4Prp/0V16+fAFyE+gN2cPmw5Rx1VmD4V4CIiERj3yO28UnEWp/IaZbQzkF38NQ/y2EdnQFNTQfrUdeAiIlF47TVOaGvkVabQgWE4BjDwAHj3XRg7NvIue3wGbmblZva2mT2T+f4wM1tiZqvN7DdmFnIFpIhIP3HssX++T75sX3hDcG3hhAkF6bI3Uyg3AJ3fS70VuMPdjwC2AVdFWZiISKoMHhy+CPjJJxc3wM1sLPA/gF9lvjfgLODRzEPmAhcVokARkcTbswfuvTf82LBhBeu2p2fgdwI/4Iu3Vw8Etrv7vn2Xm4AxEdcmIpIO+9voctmygnXbbYCb2TeAre7+ZufmkId6jufPMrNGM2tsjnI7ZhH5kvnz4bjjYOTIYOGkVauKXVE/ctBB0NYWfuzwwwvWbU/OwKcAF5jZBmABwdTJncBQM9t3FctYYHPYk919trvXu3t9XV1dBCWLSFe33AJXXw3Ll0NzMzzxRLD/4urVxa6sn6ithZkzg0W/O6upgZtuKli33Qa4u/+9u4919/HA5cDv3P0K4PfApZmHzQSeLFiVIpLTrl3w058Gn/dx/6JdYnL33TBrVhDaVVVw8MHw0ENw5pkF67IvN/L8HfC/zWwNwZz4A9GUJCK9sW4dlJdnt7e3w6uvxl9Pv1VZCXfcAdu2BdvzNDXB9OkF7bJXN/K4+0vAS5mv1wEN0ZckIr3xxz/m3oOxgNOvkktVFQwfHktXupVeJMUWLIArrwymTLqqqYEf/Sj+miQ+CnCRlOrogL/5my/Pfe9TVQUPPBDskC6lSwEuklLbtsEnn4QfGzgQLr883nokfgpwkZSqrYWKHO9ijR4dby1SHApwkZSqqoITTww/dv318dYixaEAF0mpzz6DN9/Mbi8rgw0bYi9HikABLpJSK1YE+y121dEBixbFX4/ETwEuklKjR0NLS3a7GRxySPz1SPwU4CIpNXJkENZdlZfDjTfGX4/ETwEuklLz5uU+dthh8dUhxaMAF0mphQth9+7s9poaWLw4/nokfgpwkZQaNy58EauOjmB5ail9CnCRlPqLvwhWHOzqgANgypT465H4KcBFUur228Pb6+rC39yU0qMAF0mpsJt4INiVR/oHBbhIStXW9q5dSo8CXCSlwtZBMQt29ZL+QQEukkKffBJ+qWBZWXB1ivQPCnCRFFq8OHwdlPZ2eOaZ+OuR4lCAi6TQ8OHB9d5dmcGoUfHXI8WhABdJocmTg/XAw1x9dby1SPEowEVSaPv28J3oKyuDXeqlf1CAi6TQf/4nDBiQ3b53Lzz2WPz1SHEowEVSaOhQcM9uN4MDD4y/HikOBbhICp12WvhCVroOvH9RgIuk0LZtsGtXdnt5OTQ1xV+PFIcCXCSFXn45fA68tRUefTT+eqQ4FOAiKVRbG76UbFlZMD8u/YMCXCSFtmyBzz/Pbq+u1hx4f6IAF0mZ3bvh2mvDj33jG9DQEG89UjwKcJGUWbIkmCoJ09wcby1SXApwkRRpaYFXXgm/AgVgyJB465HiqujuAWY2AHgFqM48/lF3/7GZHQYsAIYDbwFXuvveQhYr0p/t3Amnngrr1kFbW/bxQYPgmmvir0uKpydn4C3AWe5+PHACMM3MJgO3Ane4+xHANuCqwpUpIrffDmvWZJ99mwWXFF5/PZx3XnFqk+LoNsA9sG/ZnMrMhwNnAfuuOJ0LXFSQCkUEgPnzYc+e7PaqKli4EH72s/hrkuLqdgoFwMzKgTeBicC/AWuB7e6+7w+5JmBMjufOAmYBHHLIIX2tV0pUUxPMmwdbt8I558C0abnfqOuvBg4Mby8rg/HjYy1FEqJHvyLu3u7uJwBjgQbgmLCH5XjubHevd/f6urq6/CuVkvXcc3DUUfDP/wz/+q/wrW/B1KnBXYXyhWuugZqaL7eVlQX/7Q49tDg1SXH16hzH3bcDLwGTgaFmtu8MfiywOdrSpD9obYUZM4J53ZaWoG3nzuBSublzi1tb0lx9NVx4YXAmXlMT3I05erRune/Pug1wM6szs6GZrwcCU4H3gN8Dl2YeNhN4slBFSul6443wrcF27YJ///f460my8nJ4+OHgv9ldd8GCBbBhA0yYUOzKpFh6Mgc+GpibmQcvAx5x92fM7F1ggZndDLwNPFDAOqVEVVaGr2sNubcM6+8mTQo+RLoNcHd/BzgxpH0dwXy4SN5OPjl8XeuKCvjud+OvRyRN9D6/FNX27bnvKtR73iL7pwCXonr++WAFva7a24M5XhHJTQEuRRU2fbJPRY/uUhDpvxTgUlTTpn1x+WBn5eXw7W/HX49ImijApaja23NfhRK244yIfEEBLkW1cGH43o7t7cE1zyKSmwJcRCSlFOBSVOeeG2wR1lVVVXCLvYjkpgCXonrnnfBVB9vb4cSs28dEpDMFuBTVvHnhqw7W1MCiRfHXI5ImCnApqrY2XYUiki8FuBTVt74VfjPP3r3BmuAikpsCXIpq48bw9mHDtMO6SHcU4FJU998fPlWyfTusXRt/PSJpogCXotq7N7y9rCz3MREJKMClqK64InwOvLYWjj46/npE0kQBLkVVUxO+pVptLZjFX49ImijApajmzAm/jHDTpuBDRHJTgEtRhd1GD8Ec+J498dYikjYKcCmqyy4Lv5V+6FCYODH+ekTSRAEuRVVTEz6FctBBmgMX6Y4CXIrqwQfDA3zFCvjkk/jrEUkTBbgUVa4d6TUHLtI9BbgU1Te/GT5VMmYMjB4dfz0iaaIAl6JyD59CmTRJc+Ai3VGAS1EtWBDe/sILuZeZFZGAAlyKKtd14C0tCnCR7ijApaiOOSa8fcqU8OvDReQL+hWRolm7Ft5/P/zYL34Rby0iaaQAl6J56KHwhawGDYL162MvRyR1FOBSNB9/HL6hsXuwoYOI7F+3AW5m48zs92b2npmtNLMbMu3DzewFM1ud+Tys8OVKqXjjjdxXoHR0wNe/Hm89ImnUkzPwNuBv3f0YYDJwnZl9Bfgh8KK7HwG8mPlepFu7dsHZZ8O2bdnHamrguuvg8MPjr0skbboNcHf/0N3fynz9GfAeMAa4EJibedhc4KJCFSml5amnwue+y8pg+nT4+c/jr0kkjXo1B25m44ETgSXAKHf/EIKQB0bmeM4sM2s0s8bm5ua+VZtS7rB8OSxZon0eIVikqq0tu72jAw44QHdgivRUjwPczAYD/w/4X+6+o6fPc/fZ7l7v7vV1dXX51Jhqq1bBkUfCqafCOefAqFHw5JPFrqq4zjwzvH3wYJg2Ld5aRNKsRwFuZpUE4f1rd38s07zFzEZnjo8GthamxPRqawvCau1a+Pxz2LEjuLpixgxYs6bY1RXPMcfAlVcGlwvuM2gQTJ4M555bvLpE0qYnV6EY8ADwnrt3vr3iKWBm5uuZQD8/r8y2aFEQ3F1vCW9thdmzi1NTUtx7L8ybB+edF/xP7u674dlndfelSG9U9OAxU4ArgeVmtjTT9g/ALcAjZnYVsBGYXpgS06u5OXw9j9ZW2Lw5/nqSxAwuuST4EJH8dBvg7v7/gVxvK+lq3f3YsQN27sxuHzwYzj8//npEpLToD9YCeecd+MEPstvNgjc1L700/ppEpLQowAvk7ruDJVG7qqyEO++Eqqr4axKR0qIAL5CNG6G9Pbt9wACt8yEi0VCAF8j554efZbe0QEND/PWISOlRgBfISSeFr7Q3bFhwM4+ISF8pwAvk/vvD2z/9FFaujLcWESlNCvACWbcu/Brwqir48MP46xGR0qMAL4Bf/CJY7zpMSwuceGK89YhIaerJnZjSC/Pnwz/+Y/iqg4MGwQ03wIEHxl+XiJQeBXjEbr452LCgKzO4775gISsRkShoCiViuea3q6qC5WS11rWIREUBHrFTTglvHzpUUyciEi0FeMROPz28/dRTtVSqiERLkRKxuXPD2xctCr+1XkQkXwrwiOWaA9+zJ9jcQUQkKgrwiB13XHj7yJFQWxtvLSJS2hTgEfvqV8Pbzz5bV6CISLQU4BH79a/D2597Lt46RKT0KcAjlmsOfOtW6OiItxYRKW0K8IgdeWR4+4QJuoxQRKKlSInYIYeEt8+aFW8dIlL6FOAR2r4dXn45/Ngrr8Rbi4iUPgV4hJqagk2Lw6xaFW8tIlL6FOARGj8+fBs1Mzj55NjLEZESpwCP0ODBMHZsdrs7XHVV/PWISGlTgEdo7VrYtCm7vaICnn02/npEpLQpwCO0ejUMGJDd3tYGy5bFX4+IlDYFeISOPjpYtKqrykqor4+/HhEpbQrwCB16KAwcmN3e1gbf+U789YhIaVOAR+gPfwi/CmXAgNzXh4uI5EsBHqF33w1f72T3bli6NP56RKS0dRvgZjbHzLaa2YpObcPN7AUzW535PKywZaZDbS3s3ZvdPmhQ7nXCRUTy1ZMz8IeAaV3afgi86O5HAC9mvu/Xtm6F738/fNu0mhr49rfjr0lESlu3Ae7urwCfdGm+ENi3++Nc4KKI60qde+4Jpkq6KiuDBQtgyJD4axKR0pbvHPgod/8QIPN5ZHQlpdPrr0NLS3b74MHw6afx1yMipa/gb2Ka2SwzazSzxubm5l4/f80auPhiOOAAGDcObrstmbu7H3ccVFVlt7e15V4jXESkL/IN8C1mNhog83lrrge6+2x3r3f3+rq6ul51snkznHIKPPUU7NgRrPb3k5/Ad7+bZ9UFdN11UF395bbqamhogEmTilOTiJS2fAP8KWBm5uuZwJPRlPNld90Fu3Z9+dK8Xbtg/nz44INC9Ji/ceOCa71POSWY966uhhkz4Omni12ZiJSqiu4eYGbzga8BI8ysCfgxcAvwiJldBWwEpheiuFdfDb8sr7oaVq6EMWMK0Wv+TjwxmAtvbYXycm2hJiKF1W2Au/tf5Tj09YhryTJpUhDiXee89+6Fww8vdO/5y7Wpg4hIlBJ9jnjhheFvWB58MEycGH89IiJJkugAf/jh8GmIzZvh88/jr0dEJEkSHeCLF4evLVJRAevXx1+PiEiSJDrAJ0wIb9+7F0aPjrcWEZGkSXSAX3FFePvxx8OBB8Zbi4hI0iQ6wB9/PLz9/feDOxxFRPqzRAf44sXh7a2tybuRR0QkbokO8HHjwts7OjSFIiKS6ACfOTO8ffLkYJU/EZH+LNEBvmhRePvy5clckVBEJE6JDvBXXw1v37ULtmyJtxYRkaRJdIDnutbbHYYOjbcWEZGkSXSAz5gR3n7GGcE+kyIi/VmiA/z558Pb//jHeOsQEUmiRAf4kiXh7Zs2aTErEZFEB3iua72rqmDAgHhrERFJmkQHeK458PPOC3a8ERHpzxId4K+/Ht6+YkW8dYiIJFGiAzzXWiirV8OePfHWIiKSNIkO8GHDwturq4N5cBGR/izRAX7ZZeHt3/ymdnwXEUl0DDY2hrevWRNvHSIiSZToAM91Hfjbb4fvlSki0p8kOsBzLRk7aJCmUEREEh2D55wT3p5rr0wRkf4ksQHe0QG/+112u5nOvkVEIMEBvn59+Hon7vDss/HXIyKSNIkN8CFDcu+6k+v6cBGR/iSxAV5XBxMnZrebwbXXxl+PiEjSJDbA9+yBDRuy28vK4NNPYy9HRCRxEhvgb70VvuJgezs8+mj89YiIJE1iA3x/c+DDh8dbi4hIEvUpwM1smpm9b2ZrzOyHURUFMGlSsGhVV+XlcN11UfYkIpJOeQe4mZUD/wacB3wF+Csz+0pUha1eHX4ZYVlZ7p16RET6k76cgTcAa9x9nbvvBRYAF0ZTFixcGFxx0lVbGzz9dFS9iIikV18CfAywqdP3TZm2LzGzWWbWaGaNzc3NPX7xQYPC38SsrITa2t4XKyJSavoS4CHnx3hWg/tsd6939/q6uroev/gllwR3XXZVXp57r0wRkf6kLwHeBIzr9P1YYHPfyvnCsGHw2GPBmfiQIcFZ98CB8KtfwaGHRtWLiEh6VfThuW8AR5jZYcAHwOVApOfG554LW7fCokXB3PfUqUGYi4hIHwLc3dvM7PvA80A5MMfdV0ZWWUZNDVxwQdSvKiKSfn05A8fdnwW0NqCISBEk9k5MERHZPwW4iEhKKcBFRFJKAS4iklLmYXfLFKozs2bgT7182gjgowKUUwylMhaNI1lKZRxQOmOJehyHunvWnZCxBng+zKzR3euLXUcUSmUsGkeylMo4oHTGEtc4NIUiIpJSCnARkZRKQ4DPLnYBESqVsWgcyVIq44DSGUss40j8HLiIiIRLwxm4iIiEUICLiKRUrAFuZnPMbKuZrchx/Gtm9qmZLc183NTp2AYzW55pb+zUPt3MVppZh5nFcvlRgcbxczNbZWbvmNnjZjY0xWP5aWYcS83st2Z2cBrH0en4jWbmZjaikGPI9FWIn8c/mdkHnZ5zfhrHkTl2fWYj9ZVmdluhx5HpsxA/k990evwGM1uaV3HuHtsHcDpwErAix/GvAc/kOLYBGBHSfgxwFPASUJ/icZwDVGS+vhW4NcVjGdLp6/8J3JvGcWSOjSNYMvlPuR6T9HEA/wTcGMe/pwKP40xgEVCd+X5kWsfS5TG3AzflU1usZ+Du/grwScSv+Z67vx/la/agz0KM47fu3pb59jWCHY4KrkBj2dHp20GEbLUXtUKMI+MO4AfEMAYo6DhiVaBxXAvc4u4tmT62Rvz6oQr5MzEzAy4D5ufz/CTOgZ9qZsvM7Dkzm9Sp3YHfmtmbZjarWMX1Ql/G8R3gucKX2GO9HouZ/YuZbQKuAG4iGXo1DjO7APjA3ZfFXun+5fNv6/uZaa05ZjYsxlr3p7fjOBL4qpktMbOXzeyUeMvdr3x/378KbHH31Xn1GuefVZk/F8aT+0+RIcDgzNfnA6s7HTt4359NwDLg9C7PfYmYplAKPI4fAY+TucQzzWPJHPt74CdpGwdQAywBDsgc20AMUyiF+HkAowh2zSoD/oVg96w0jmMFcBfBhuoNwPq4fk8K+Pv+S+Bv860rUWfg7r7D3Xdmvn4WqNz3xpG7b8583koQcA1FK7Qb+Y7DzGYC3wCu8MxPt9gi+Jk8DPxlTOXmlMc4JgCHAcvMbAPBlNZbZnZQEcr/s3x+Hu6+xd3b3b0DuJ8E/O7k+e+qCXjMA68DHQSLRhVVH37fK4BLgN/k23eiAtzMDsrMCWFmDQT1fWxmg8ysNtM+iOANv9B3hJMgn3GY2TTg74AL3H1XcSrPludYjuj0EhcAq+KtOltvx+Huy919pLuPd/fxBOFxkrv/V5GGQKbGfH4eozu9xMUk4Hcnz9/1J4CzMseOBKpIwMqFfcitqcAqd2/Kt+8+7YnZW2Y2n+Ad2xFm1gT8GKgEcPd7gUuBa82sDdgNXO7ubmajgMcz/40qgIfdfWHmNS8G7gbqgP8ws6Xufm7axgH8H6AaeCFz/DV3v6aQ4yjgWG4xs6MIzpD+BKR1HLEr0DhuM7MTCOZjNwDfS+k45gBzLLicby8wM46/VAv4b+ty8nzz8s+1JeQvdRER6aVETaGIiEjPKcBFRFJKAS4iklIKcBGRlFKAi4j0kXWz4FWXx55uZm+ZWZuZXdrl2G0WLNT1npndte/yxFwU4CIiffcQMK2Hj90I/DXBTW5/ZmanAVOA44BjgVOAM/b3QgpwEZE+8pAFr8xsgpktzKyD8gczOzrz2A3u/g7BfRJfehlgAMENStUE15pv2V+/CnARkcKYDVzv7icDNwL37O/B7r4Y+D3wYebjeXd/b3/PifVOTBGR/sDMBgOnAf+30zR2dTfPmUiwv8G+paRfMLPTM2f3oRTgIiLRKwO2u/sJvXjOxQRLaOwEMLPngMlAzgDXFIqISMQ82NRkvZlNh2DjBjM7vpunbQTOMLMKM6skeANzv1MoCnARkT7KLHi1GDjKzJrM7CqCzUyuMrNlwErgwsxjT8ksijUduM/MVmZe5lFgLbCcYO3wZe7+9H771WJWIiLppDNwEZGUUoCLiKSUAlxEJKUU4CIiKaUAFxFJKQW4iEhKKcBFRFLqvwGEkBehisp4+wAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "y = range(0, len(s19_times.index))\n",
+ "colors = s19_times[PS2.ProblemID].isin(dropped_problems).apply(lambda x: 'red' if x else 'blue')\n",
+ "plt.scatter(s19_times['TimeInt'], y, color=colors)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<matplotlib.collections.PathCollection at 0x2102af98708>"
+ ]
+ },
+ "execution_count": 29,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEFCAYAAAD69rxNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAcIElEQVR4nO3de5BV5Znv8e/TN6C7QW4toFFbsL0mUbFjFOMNMxkdjWKiU1HrBI+Oxso4mUwdnRCP5mRqLmXMVMxkciaGo5kwmYyXMbEgVDRBRo1xItoIaLgoIojIrUGQe0PTz/ljbWLb/S76tvbae63+fap27d7v2t3reavl5+p3r/d9zd0REZHsqSh1ASIi0j8KcBGRjFKAi4hklAJcRCSjFOAiIhlVlebJxo4d642NjWmeUkQk8xYuXLjF3Ru6tqca4I2NjbS0tKR5ShGRzDOzt0PtGkIREckoBbiISEb1KsDNbKSZPW5mK8xsuZmda2ajzWyema0sPI8qdrEiIvKB3l6B/xPwlLufDJwOLAdmAPPdvQmYX3gtIiIp6THAzWwEcAHwEIC773f37cBVwKzC22YB04pVpIhIVm3eDAsXws6dyf/s3lyBTwRagX81s0Vm9qCZ1QHj3H0DQOH5yNA3m9mtZtZiZi2tra2JFS4iUs727YMvfAGOPRamToUjj4S774Yk1w/sTYBXAZOBH7j7mcBu+jBc4u4z3b3Z3ZsbGrrdxigikktf+QrMmQNtbbBjRxTo3/0uPPRQcufoTYCvA9a5+4LC68eJAn2TmU0AKDxvTq4sEZHsamuDn/wE9u79cPvu3XDffcmdp8cAd/eNwDtmdlKh6RJgGTAHmF5omw7MTq4sEZHs2rkTOjrCx7ZsSe48vZ2J+RfAT82sBngL+J9E4f+Ymd0MrAWuTa4sEZHsGjMGKmIuj889N7nz9CrA3X0x0Bw4dElypYiI5MOyZfFX4EkGuGZiiogk7KWXoLo6fGzlyuTOowAXEUnYscdCe3v39qoqOOGE5M6jABcRSdikSbB/f/f29nb41KeSO48CXEQkYXPnQk1N9/bKSnj++eTOowAXEUmYWXjGpVn0SIoCXEQkYSeeGB5COXgQrk3whmsFuIhIwh55JHylXVkJBw4kdx4FuIhIwlavDg+h1NXBu+8mdx4FuIhIwo46Kty+dy9MnpzceRTgIiIJcodnngkfmzAhWlY2KQpwEZEEvf8+xG198N57yZ5LAS4ikqC6uujDypCkt0RQgIuIJKi6GkaPDh878cRkz6UAFxFJ0I4d0T6YIYsXJ3suBbiISII6OuJnW8YtMdtfCnARkQSNHAnDh4ePnX9+sudSgIuIJGjr1mhLtRDdhSIiUsY2bAivRAiwdm2y51KAi4gkaMeOaPf5rioqYMqUZM+lABcRSdBdd8Ufu+eeZM+lABcRSdCLL4bbOzqgsTHZcynARUQSNHL4wWB7ba1TVZXsuRTgIiIJ+mrtD6nlw4Pgw9jDlya3JLobDyjARUSSs2ULd268g+nMYih7OYLtDGUvn+Pn3Lv+i4mfrlcX9Ga2BtgJHATa3b3ZzEYDjwKNwBrgT919W+IViohkxYEDVFoH/8Kf87fcw0qaOJ7VjGMzHDgm8dP15Qr8Ync/w92bC69nAPPdvQmYX3gtIjJ4NTT8Yb78GN7jHBZE4Q1w1VWJn24gQyhXAbMKX88Cpg28HBGRDHvqqWg5wq7MYOLExE/X2wB34NdmttDMbi20jXP3DQCF5+A+E2Z2q5m1mFlLa9wq5yIiSXGHOXPgiivg4ovhwQfDW8QXw6pV0dbzoZreeSfx0/X2ppbz3H29mR0JzDOzFb09gbvPBGYCNDc3B7b5FBFJ0F/9VRTah6ZDvvwy/Nu/Rfucxe20kJQzzoiuwNvaPtxeXw9nnZX46Xp1Be7u6wvPm4EngLOBTWY2AaDwHLMCrohISlatgh/+8MNz2XfvhkWLoqvyYrvgAjj1VBgy5IO26upobPyaaxI/XY8BbmZ1Zjb80NfAZ4DfA3OA6YW3TQdmJ16diEhvtbfD978fDVd0tWsXPPlk8Wswg//6L/jyl6PQHjUKvvhFeOmlD4d6QnozhDIOeMKiO9CrgP9w96fM7GXgMTO7GVgLXJt4dSIivbFsGUydGu0o3HX4Aj64Ck5DXR185zvRo8h6DHB3fws4PdC+FbikGEWJiPSaO1x+OWzaFP+eqiq46ab0akqJZmKKSLYtWgRbtoSPVVVF2+P8+7/DpEnp1pWChJdWERFJWWjx7UNOOSW6C6UI48/lQFfgIpJtH/1oOMTN4JZbchveoAAXkaybOze8h5kZjB+ffj0pUoCLSLa9/nr4zhMzWL06/XpSpAAXkWw77bTwDMuKiuhYjinARSTbRo78wwqAH9Lenss7TzpTgItIts2bF559OWRINCsyxxTgIpJto0bFD6EccUT69aRIAS4i2TZlSngJ1z174DOfSb+eFCnARSTbnn46utruatgweO659OtJkQJcRLJt27bwh5gVFdHiVjmmABeRbDv77HD73r3w6U+nW0vKFOAikm2vvBJN2unKLFrMKscU4CKSbQsXhm8jrK+PZmnmmAJcRLLtlFPC7Xv3QlNTurWkTAEuItk2YkT8saOOSq+OElCAi0i2Pf10uL26OtpqLccU4CKSbfX14faDB6P9KXNMAS4i2XbmmeH2qiqYODHdWlKmABeRbFuwINy+fz+0tqZbS8oU4CKSbdu2hdurqzUTU0SkrI0ZE24fMgSOPz7dWlKmABeR7Nq9G5YuDR879dTwMrM5ogAXkexavTp+uvzGjenWUgK9DnAzqzSzRWY2t/D6eDNbYGYrzexRMwtsCy0iUkRr18KuXeFjJ56Ybi0l0Jcr8L8Elnd6/S3gfndvArYBNydZmIjIYbnDl74UPlZdDXffnW49JdCrADezjwCXAw8WXhswFXi88JZZwLRiFCgivbd5M/zud7m/ey7yxhvxd6CMGwfnnptuPSXQ2yvw7wJ/DRxaNX0MsN3d2wuv1wFHh77RzG41sxYza2kdFP9ViaTvwAG48UY49li47LLo+c/+LNqYPbeGDAlv5ADxd6bkTI8BbmZXAJvdfWHn5sBbA+s5grvPdPdmd29uaGjoZ5kicjj33AOPPQZtbdGtz/v2wcMPwz/8Q6krK6LGRjjhhO5rgdfWwm23laSktPXmCvw84EozWwM8QjR08l1gpJkd+vj3I8D6olQoIj36wQ+i1VM727MHvve90tSTmp/9DMaPh+HDo+AeNgyuuAJuuaXUlaWix+0q3P3rwNcBzOwi4A53v8HM/hO4hijUpwOzi1iniMRwh507w8e2b0+3ltQ1NcHbb8NTT8H69dEO9R/7WKmrSs1A9hv6GvCImf0dsAh4KJmSRKQvzOC442DNmu7HTj459XLSV10Nn/1sqasoiT4FuLs/Czxb+PotIGY3URFJi3v8rdA5n4g46GkmpkjG7doVP1SyalW6tUi6FOAiGVdbG40ihEyYkG4tki4FuEjGHTgQbT4Tcuml6dYi6VKAi2Tc889Hc1pCQh9sSn4owEUyrrIy/go8bmhF8kEBLpJxo0ZFk3a6Moum10t+KcBFMu6BB6Ai8C+5slIfYuadAlwk49auDa/pVFs7KPY0GNQU4CIZd9pp4fa9e6G5Od1aJF0KcJGMe/XVcHtVVbQstuSXAlwk4xYuDLcfPAhbtqRbi6RLAS6ScWPHhtsrKmDEiHRrkXQpwEUybuTIcPsxx8DQoenWIulSgItkmDssWhQ+tnZturVI+hTgIhnnwc0MZTBQgItkmFn8OPfFF6dbi6RPAS6SYStWhKfRQ/zYuOSHAlwkw1asiB9CeffddGuR9CnARTJq3z742tegra37saFDNYQyGCjARTLq0Ufjr7Lr6uD229OtR9KnABfJqF/9Cnbv7t5eVQXf+AY0NKRfk6RLAS6SUePHh9uHDoVTT023FikNBbhIRm3bFm5vb9f492ChABfJqF/8Itze3h49JP96DHAzG2pmL5nZEjNbamZ/U2g/3swWmNlKM3vUzGqKX66IHLJ/f/yx0AYPkj+9uQJvA6a6++nAGcClZnYO8C3gfndvArYBNxevTBHpbO9eOHAgfOzss2HYsHTrkdLoMcA9sqvwsrrwcGAq8HihfRYwrSgVikg3P/95tOdlyDT9Sxw0ejUGbmaVZrYY2AzMA1YB29390EjbOuDomO+91cxazKyltbU1iZpF/mDDBrjzTvjEJ+C66+I3N8ib1aujq/CQffvSrUVKp6o3b3L3g8AZZjYSeAI4JfS2mO+dCcwEaG5u1rppkpi334bJk2HXrmg8+JVXYM4cePhhuPLKUldXXJMnR5sW79r14fb6+uiYDA59ugvF3bcDzwLnACPN7ND/AD4CrE+2NJHDu+ceeP/9Dz7M6+iIFna67bb8f4j3x38MkybBkCEftA0ZAhMnwqWXlq4uSVdv7kJpKFx5Y2bDgE8Dy4FngGsKb5sOzC5WkSIh8+dH+z52tX17/hdyqqyE55+PpsuPGxc9vvzlqC1ubFzypzdDKBOAWWZWSRT4j7n7XDNbBjxiZn8HLAIeKmKdIt0MHx5u7+gYHHtBDh8O//iP0UMGpx4D3N1fBc4MtL8FnF2MokR6I24Z1aYmOOKIdGsRKQXNxJRM2roV1qwJH4vb4EAkbxTgkkltbdF2YiGHm6EokicKcMmk2tr4mYif+1y6tYiUigJcMumnP4Xq6u7tZnDBBenXI1IKCnDJpGXLwluJ1dTApk3p1yNSCgpwyaTTT4eKwH+97vCxj6Vfj0gpKMAls0K3ER44ACedlH4tIqWgAJdMevLJcIDX18Nvf5t+PSKloACXzNm4EZ59Nv74qFGplSJSUgpwyZzrr4edO8PH6ut1F4oMHgpwyZQtW+C//zu8iFVVFcybp8WcZPBQgEum7NkTv1TsuHFw2mnp1iNSSgpwyZSWlvCO65WVmoEpg48CXDLlrrvCd5+4w913p1+PSCkpwCVT4lYg7OiAkSNTLUWk5BTgkilHB7fOhjFjwmujiOSZAlwyJe4e75Ej45eXFckrBbhkytKl4fbVq8MfborkmQJcMsM9fhu16mrd/y2DjwJcMuPRR8MTeMyi2ZkaQpHBRgEumXH//fHDJDNmpFuLSDlQgEtmbNsWbq+rC1+Zi+SdAlwy45OfDLd3dMCJJ6Zbi0g5UIBLZmzYEG7v6Ijf4FgkzxTgkhnLl4fbKyth/fp0axEpBz0GuJkdY2bPmNlyM1tqZn9ZaB9tZvPMbGXhWcvoS1EddVS4/eBBGD8+3VpEykFvrsDbgf/l7qcA5wB/bmanAjOA+e7eBMwvvBYpmq1bw+1NTVBbm24tIuWgxwB39w3u/krh653AcuBo4CpgVuFts4BpxSoyy/bvjzYgWLQofhKK9GzbNli3LnxMwycyWPVpDNzMGoEzgQXAOHffAFHIA0fGfM+tZtZiZi2tra0DqzZjnngCGhrgssuibb4mTYqfCi6Hd7iFqoYNS68OkXLS6wA3s3rgZ8BX3X1Hb7/P3We6e7O7Nzc0NPSnxkxauRJuuAF27Igeu3ZF63VMnao7Jvqjvh4uuSTaNq2zYcPglltKU5NIqfUqwM2smii8f+ruPy80bzKzCYXjE4DNxSkxmx58MDxrcO/eaN9G6bsf/zj6K2b48GjyTm0tXHihZmHK4FXV0xvMzICHgOXu/p1Oh+YA04F7C8+zi1JhRr37bvhK2z3+wzg5vHHjYNky+M1vor9mzjwTzjij1FWJlE6PAQ6cB/wP4DUzW1xou4souB8zs5uBtcC1xSkxm5YsCbe3t8P556dbS55UVMBFF0UPkcGuxwB3998Cceu8XZJsOfmwcCG89Vb42HnnQWNjquWISE5pJmYRLFoUf+y449KrQ0TyTQFeBMcdF93/3VVNDZx8cvr1iEg+KcCLYOfO8PKm+/fDNdekX4+I5JMCvAgeeSQ867KuDl57Lf16RCSfFOBFUFMTbq+oiD8mItJXCvAiiLvLZN8+uPjiVEsRkRxTgBfB3Lnxx9ra0qtDRPJNAV4Em2MWFaiqgu3b061FRPJLAV4EEyeG24cMgaOPTrcWEckvBXjC3OHNN8PHJkyItv8SEUmCAjxhO3bAe++Fj73zTrq1iEi+KcATVlcXf5WtfRtFJEkK8IQdOBCehQlw+eXp1iIi+aYAT9hzz8HQoeFjGkIRkSQpwBNmFt6JB6KZmCIiSVGkJGzUqGjbtK4qKuDGG1MvR0RyTAGesAceiK7CuzLTWuAikiwFeII6OuCFF8IrEdbXR/tkiogkRQGeoJtuit9Kra0t2oRXRCQpCvCEvPEGPPZY+APMmhq4/XY48sj06xKR/FKAJ+TFF+Mn8DQ3w333pVuPiOSfAjwh7rB7d/f2mppoDfDQB5siIgOhAE/I978f/vDSHW65Jf16RCT/FOAJ2Lw5fq/L0aN1+6CIFEePAW5mPzKzzWb2+05to81snpmtLDyPKm6Z5a2jI3qEDBmSbi0iMnj05gr8x8ClXdpmAPPdvQmYX3g9aK1eHb77pKICrr8+/XpEZHDoMcDd/TdA1xWurwJmFb6eBUxLuK5Muffe+PHvm25Kvx4RGRz6OwY+zt03ABSeY+9wNrNbzazFzFpaW1v7ebroDo9vfxvOOgsuvDC65zoUmqWwalW4vb5ee2CKSPFUFfsE7j4TmAnQ3Nzcr8hta4MpU2Dlyg8Wilq4EJ5/Hv75nxMrtd8+9Sl4/fXuwyjt7XDyyaWpSUTyr79X4JvMbAJA4TlmH/ZkPPpodJXbeZW/3bvhwQdhzZpinrl3ZsyIduLpvFxsbS3ceScMH166ukQk3/ob4HOA6YWvpwOzkyknbO7c8CSZqqpo8ahSa2yEl1+Gz38+mi5/2mnRqoTf/GapKxORPOtxCMXMHgYuAsaa2Trg/wD3Ao+Z2c3AWuDaYhX4wgswO+Z/D2bls75IU1M0Li8ikpYeA9zdr4s5dEnCtXRz4AB89rOwf3/3Y2YwYgRMnVrsKkREylNZz8R87rn4DYLr6+GZZ+IXkBIRybuyDvD9++NvFbz44mjYQkRksCrrAD//fNi1K3zsiivSrUVEpNyUdYAvXhwtx9pVRQUMYE6QiEgulHWAr1oV3SrYVUdHNHFGRGQwK+sAP+EE2LOne3tNDXzyk+nXIyJSTso6wBcsCO9ks38/TBvUy2eJiJR5gM+eHV5ne/hwWLEi/XpERMpJWQd4Q0O4vb0dxoxJtxYRkXJT1gF+wgnhdnf4+MfTrUVEpNyUdYA//XS43R3WrUu3FhGRclPWAR43iaeqKnx3iojIYFLWAX7WWeF2M02jFxEp6wB/441w+8GD8YtciYgMFmUd4KtXh9vdYevWdGsRESk3ZR3gEyeG26uqYOzYdGsRESk3ZR3gQ4eG20eMCK+RIiIymJR1gC9eHG7fuPHDGxyLiAxGZR3gtbXh9qoqqK5OtxYRkXJT1gF+3HHh9okTNYQiIlLWAb50abhdszBFRMo4wN1h377wMY1/i4iUcYCbRXebhEyZkm4tIiLlqGwD/LXX4q/AJ09OtxYRkXJUtgG+bFn8nSZr16Zbi4hIORpQgJvZpWb2upm9aWYzkioK4KSTwlfglZVw+ulJnklEJJv6HeBmVgn8X+Ay4FTgOjM7NanC6uqinXe6OngQmpuTOouISHYN5Ar8bOBNd3/L3fcDjwBXJVMWPPlktPt8yAsvJHUWEZHsGkiAHw280+n1ukLbh5jZrWbWYmYtra2tvf7hdXXhyTo1NVBf3/diRUTyZiABboE279bgPtPdm929uSFul+KAq6+O7gXvqrISrr++L2WKiOTTQAJ8HXBMp9cfAdYPrJwPjB4Njz8eXYkPHx49hg2DmTOhsTGps4iIZNdAVhR5GWgys+OBd4EvAIleG192GWzaBPPmRR9o/tEfwRFHJHkGEZHs6neAu3u7md0O/AqoBH7k7jGrl/RfXR1Mm5b0TxURyb4Brenn7r8EfplQLSIi0gdlOxNTREQOTwEuIpJRCnARkYxSgIuIZJR5aLZMsU5m1gq8ncKpxgJbUjhPKahv2aS+ZVO59O04d+82EzLVAE+LmbW4ey6XvFLfskl9y6Zy75uGUEREMkoBLiKSUXkN8JmlLqCI1LdsUt+yqaz7lssxcBGRwSCvV+AiIrmnABcRyaiyD3Az+5GZbTaz38ccv8jM3jezxYXHNwrtJ3VqW2xmO8zsq4Vjo81snpmtLDyPSrNPnWovRt+uNbOlZtZhZiW7/alIffu2ma0ws1fN7AkzG5lmnzrVXoy+/W2hX4vN7NdmdlSafepUe+J96/S9d5iZm9nYNPoSqL0Yv7dvmtm7nY79SZp9wt3L+gFcAEwGfh9z/CJgbg8/oxLYSHQzPMB9wIzC1zOAb+Wob6cAJwHPAs05+719BqgqfP2tnP3eRnQ69hXggbz0rdB2DNHS028DY/PSN+CbwB2l6I+7l/8VuLv/BnhvgD/mEmCVux+aBXoVMKvw9SygJCuOF6Nv7r7c3V8fcHEDVKS+/drd2wvHXiTaBSp1Rerbjk7H6ghsT5iGIv17A7gf+GtK1C8oat9KpuwDvJfONbMlZvakmZ0WOP4F4OFOr8e5+waAwvORaRTZT33tW5YMpG83AU8Wr7QB63PfzOzvzewd4AbgG2kU2U996puZXQm86+5LUquw//rz3+TtheGvH6U+HFuqS/8+/unTSPyfPSOA+sLXfwKs7HK8hmgtg3Gd2rZ3ec+2vPSt07FnKeEQSpH79r+BJyjcBpunvhWOfx34mzz0DagFFgBHFF6voURDKMX4vQHjiIZVKoC/J9qZLLX+ZP4K3N13uPuuwte/BKq7fEhyGfCKu2/q1LbJzCYAFJ43p1ZwH/Szb5nQ376Z2XTgCuAGL/wLKjcJ/N7+A/h8kcvsl370bRJwPLDEzNYQDXu9YmbjUyy7V/rze3P3Te5+0N07gP8HnJ1mzZkPcDMbb2ZW+Ppsoj5t7fSW6+j+J88cYHrh6+nA7GLX2R/97Fsm9KdvZnYp8DXgSnffk1atfdXPvjV1enklsKLYdfZHX/vm7q+5+5Hu3ujujcA6YLK7b0yx7F7p5+9tQqeXVwPBO1yKplR/yvThT56HgQ3AAaJf/s3AbcBtheO3A0uBJUQfbE3p9L21RL+AI7r8zDHAfGBl4Xl0jvp2deFntQGbgF/lqG9vAu8AiwuPUt2pUYy+/YzoH/+rwC+Ao/PSty4/fw2luwulGL+3nwCvFX5vc4AJafZJU+lFRDIq80MoIiKDlQJcRCSjFOAiIhmlABcRySgFuIjIAPW0UFaX915gZq+YWbuZXdPl2H0WLUa33My+d+i2xjgKcBGRgfsxcGkv37sWuJFowtYfmNkU4Dzg48BHgU8AFx7uBynARUQGyAMLZZnZJDN7yswWmtnzZnZy4b1r3P1VoKPrjwGGEk3ZHwJUE83liKUAFxEpjpnAX7j7WcAdwL8c7s3u/jvgGaLJRhuIJuEtP9z3VCVUqIiIFJhZPTAF+M9Ow9hDevieE4jW8z+0TPI8M7ugcHUfpAAXEUleBdGqp2f04XuuBl70woJaZvYkcA4QG+AaQhERSZhHG3SsNrNrASxyeg/ftha40MyqzKya6APMww6hKMBFRAbIzB4GfgecZGbrzOxmoo05bjazJUSLZF1VeO8nzGwdcC3wQzNbWvgxjwOriBbHWgIscfdfHPa8WsxKRCSbdAUuIpJRCnARkYxSgIuIZJQCXEQkoxTgIiIZpQAXEckoBbiISEb9f5Vv+sjxLQyCAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# The added assignment comes between 4 and 5 in spring\n",
+ "# We could just skip this assignment for F19->S19 prediction...\n",
+ "y = range(0, len(f19_times.index))\n",
+ "colors = f19_times[PS2.ProblemID].isin(added_problems).apply(lambda x: 'red' if x else 'blue')\n",
+ "plt.scatter(f19_times['TimeInt'], y, color=colors)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "S19 Problem 45 == F19 Problem 736\n",
+ "\n",
+ "S19 Problem 48 == f19 Problem 737"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 48,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def print_code_samples(df, ps2, problem_id, n_samples):\n",
+ " code_states = df[(df[PS2.ProblemID] == problem_id) & (df[PS2.EventType] == 'Run.Program') & (df[PS2.Score] == 1)].sample(n_samples)[PS2.CodeStateID]\n",
+ " solutions = code_states.apply(lambda cs_id: ps2.get_code_for_id(cs_id))\n",
+ " for sol in solutions:\n",
+ " print(sol)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "public int sum67(int[] nums)\r\n",
+ "{\r\n",
+ " int sum = 0;\r\n",
+ " int change = 0;\r\n",
+ " for(int i = 0; i < nums.length ; i++)\r\n",
+ " {\r\n",
+ " if(nums[i] == 6)\r\n",
+ " {\r\n",
+ " for(int c = i; c < nums.length ; c++)\r\n",
+ " {\r\n",
+ " if(nums[c] == 7)\r\n",
+ " {\r\n",
+ " change = c + 1;\r\n",
+ " break;\r\n",
+ " }\r\n",
+ " }\r\n",
+ " }\r\n",
+ " if(nums[i] == 6)\r\n",
+ " i = change;\r\n",
+ " if(i < nums.length)\r\n",
+ " {\r\n",
+ " \tsum = sum + nums[i];\r\n",
+ " }\r\n",
+ " }\r\n",
+ " return sum;\r\n",
+ "}\r\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "print_code_samples(s19, s19_ps2, 45, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "public int[] withoutTen(int[] nums)\r\n",
+ "{\r\n",
+ " int[] newArray = new int[nums.length];\r\n",
+ " for(int val : newArray)\r\n",
+ " val = 0;\r\n",
+ " \r\n",
+ " int newIndex = 0;\r\n",
+ " for (int i = 0; i < nums.length; i++){\r\n",
+ " \tif(nums[i] != 10)\r\n",
+ " newArray[newIndex++] = nums[i]; \r\n",
+ " }\r\n",
+ " \r\n",
+ " return newArray;\r\n",
+ "}\r\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "print_code_samples(s19, s19_ps2, 48, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 49,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "public int sum67(int[] nums)\r\n",
+ "{\r\n",
+ " boolean deadZone = false;\r\n",
+ " if (nums.length > 0)\r\n",
+ " {\r\n",
+ " int sum = 0;\r\n",
+ " for (int i = 0; i < nums.length; i++)\r\n",
+ " {\r\n",
+ " if (nums[i] == 6)\r\n",
+ " {\r\n",
+ " deadZone = true;\r\n",
+ " }\r\n",
+ " \r\n",
+ " if (deadZone == false)\r\n",
+ " {\r\n",
+ " \tsum += nums[i];\r\n",
+ " }\r\n",
+ " \r\n",
+ " if (nums[i] == 7)\r\n",
+ " {\r\n",
+ " deadZone = false;\r\n",
+ " }\r\n",
+ " }\r\n",
+ " return sum;\r\n",
+ " }\r\n",
+ " else\r\n",
+ " {\r\n",
+ " return 0;\r\n",
+ " }\r\n",
+ "}\r\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "print_code_samples(f19, f19_ps2, 736, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "public int[] withoutTen(int[] nums)\r\n",
+ "{\r\n",
+ " int[] r = new int[nums.length];\r\n",
+ "\tint j = 0;\r\n",
+ "\tfor(int i = 0; i < nums.length; i++)\r\n",
+ "\t{\r\n",
+ "\t\tif(nums[i] != 10)\r\n",
+ "\t\t{\r\n",
+ "\t\t\tr[j] = nums[i];\r\n",
+ "\t\t\tj++;\r\n",
+ "\t\t}\r\n",
+ "\t}\r\n",
+ "\t\r\n",
+ "\treturn r;\r\n",
+ "}\r\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "print_code_samples(f19, f19_ps2, 737, 1)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/eval_models.bat b/eval_models.bat
new file mode 100644
index 0000000000000000000000000000000000000000..f5b8b6d73caeb01eaa17bf844a1f86d913945bc5
--- /dev/null
+++ b/eval_models.bat
@@ -0,0 +1,4 @@
+python .\CodaLabPackages\Track1Package\program\evaluate.py .\data\Prediction\S19\basic_LR_task1\ .\data\Results\S19\basic_LR_task1
+python .\CodaLabPackages\Track1Package\program\evaluate.py .\data\Prediction\S19\code_RF_task1\ .\data\Results\S19\code_RF_task1
+python .\CodaLabPackages\Track1Package\program\evaluate.py .\data\Prediction\F19\basic_LR_task1\ .\data\Results\F19\basic_LR_task1
+python .\CodaLabPackages\Track2Package\program\evaluate.py .\data\Prediction\S19\basic_LR_task2\ .\data\Results\S19\basic_LR_task2
\ No newline at end of file
diff --git a/modelo_v1.ipynb b/modelo_v1.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..75b361bf11719010e7231bf1eea0e04408d03dc6
--- /dev/null
+++ b/modelo_v1.ipynb
@@ -0,0 +1,239 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "ca052d45",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#importando librerias\n",
+ "from sklearn import datasets\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "import sklearn.linear_model as sk\n",
+ "import numpy as np"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "id": "fe82d9b0",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[[1, 1], [2, 1], [3, 1], [1, 1], [2, 1], [11, 1], [7, 1], [7, 1], [3, 1], [2, 1], [1, 1], [2, 1], [1, 1], [6, 1], [1, 1], [9, 1], [4, 1], [18, 1], [22, 1], [45, 1], [3, 1], [37, 1], [7, 1], [30, 1], [5, 1], [28, 1], [11, 1], [13, 1], [19, 1], [16, 1], [2, 1], [4, 1], [1, 1], [1, 1], [2, 1], [5, 1], [8, 1], [3, 1], [2, 1], [2, 1], [9, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [11, 1], [1, 1], [2, 1], [2, 1], [31, 1], [1, 1], [6, 1], [17, 1], [1, 1], [24, 1], [22, 1], [10, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [11, 1], [5, 1], [2, 1], [7, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [10, 1], [21, 1], [2, 1], [1, 1], [5, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 0], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [12, 1], [6, 1], [4, 1], [7, 1], [6, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [6, 1], [3, 1], [1, 1], [5, 1], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [2, 1], [11, 1], [4, 1], [1, 1], [4, 1], [6, 1], [2, 0], [1, 1], [5, 1], [4, 1], [4, 1], [5, 1], [10, 0], [10, 1], [3, 1], [15, 0], [8, 0], [22, 1], [5, 1], [6, 0], [6, 0], [1, 1], [14, 0], [19, 1], [9, 0], [9, 0], [2, 1], [13, 1], [4, 1], [1, 1], [12, 1], [1, 1], [2, 1], [6, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [2, 0], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [8, 1], [1, 1], [6, 1], [31, 1], [15, 1], [1, 1], [2, 1], [6, 1], [2, 1], [5, 1], [6, 1], [4, 1], [25, 1], [6, 1], [3, 1], [10, 1], [2, 1], [7, 1], [40, 0], [2, 1], [3, 1], [4, 0], [6, 0], [4, 1], [17, 1], [2, 0], [8, 1], [27, 0], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [3, 1], [13, 1], [3, 1], [1, 1], [2, 1], [3, 1], [2, 1], [1, 1], [1, 1], [5, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [3, 1], [1, 1], [19, 0], [1, 1], [2, 1], [3, 1], [1, 0], [7, 1], [5, 1], [6, 0], [7, 1], [3, 0], [2, 1], [5, 1], [18, 1], [1, 0], [10, 1], [4, 1], [3, 1], [4, 1], [1, 1], [1, 1], [14, 1], [2, 1], [1, 1], [3, 1], [1, 1], [9, 1], [1, 1], [1, 1], [1, 1], [4, 1], [9, 1], [1, 1], [2, 1], [8, 1], [12, 1], [4, 1], [2, 1], [7, 1], [13, 1], [3, 1], [2, 1], [21, 1], [3, 1], [16, 1], [33, 1], [6, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [4, 1], [2, 1], [3, 1], [1, 1], [3, 1], [3, 1], [37, 1], [14, 1], [16, 1], [18, 1], [9, 0], [7, 1], [4, 1], [6, 1], [11, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 0], [1, 1], [5, 1], [5, 1], [1, 1], [16, 1], [12, 1], [3, 1], [1, 1], [3, 1], [5, 1], [1, 1], [5, 1], [6, 1], [9, 1], [15, 1], [1, 1], [8, 0], [3, 1], [3, 1], [14, 0], [1, 1], [1, 1], [1, 0], [6, 0], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [1, 1], [11, 1], [4, 1], [14, 0], [2, 1], [41, 1], [9, 1], [10, 1], [5, 1], [2, 1], [15, 1], [5, 1], [2, 1], [2, 1], [9, 0], [9, 1], [12, 0], [3, 1], [3, 1], [7, 1], [6, 1], [10, 0], [7, 1], [1, 1], [19, 0], [4, 1], [1, 1], [10, 0], [7, 1], [12, 1], [6, 1], [7, 1], [2, 1], [1, 1], [2, 1], [3, 1], [1, 1], [22, 1], [32, 1], [3, 1], [3, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [2, 1], [3, 1], [2, 1], [4, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [8, 1], [1, 1], [4, 1], [3, 1], [1, 1], [9, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [11, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 0], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [10, 0], [8, 1], [3, 1], [4, 1], [16, 1], [11, 1], [17, 0], [17, 0], [13, 0], [4, 1], [11, 1], [5, 1], [3, 0], [18, 0], [8, 1], [11, 0], [1, 0], [4, 0], [9, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [6, 0], [9, 0], [1, 0], [3, 0], [14, 0], [1, 1], [1, 1], [3, 1], [1, 1], [18, 1], [3, 1], [2, 1], [4, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [2, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [11, 1], [2, 1], [1, 1], [4, 1], [3, 1], [1, 1], [5, 1], [2, 1], [1, 1], [1, 1], [4, 1], [16, 1], [1, 1], [1, 1], [5, 1], [1, 1], [3, 1], [3, 1], [4, 1], [3, 1], [8, 1], [2, 1], [6, 1], [1, 1], [1, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [11, 1], [8, 1], [1, 1], [13, 1], [3, 1], [2, 1], [3, 1], [4, 1], [1, 1], [2, 1], [7, 1], [9, 1], [5, 1], [1, 1], [17, 1], [19, 1], [4, 1], [21, 1], [2, 1], [3, 1], [1, 1], [12, 1], [18, 0], [4, 1], [48, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [9, 1], [4, 1], [1, 1], [2, 1], [1, 1], [5, 1], [5, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [3, 1], [3, 1], [1, 1], [5, 1], [3, 1], [2, 1], [4, 1], [1, 1], [4, 1], [1, 1], [4, 1], [3, 0], [4, 1], [3, 1], [1, 1], [5, 1], [9, 0], [18, 0], [6, 1], [4, 1], [2, 0], [9, 0], [2, 1], [16, 0], [5, 1], [3, 0], [9, 1], [6, 1], [5, 1], [3, 1], [6, 1], [1, 1], [11, 1], [10, 1], [2, 1], [2, 1], [1, 1], [4, 1], [1, 1], [6, 1], [2, 1], [3, 1], [3, 1], [1, 1], [9, 1], [6, 1], [1, 1], [3, 1], [3, 1], [3, 1], [6, 0], [1, 1], [4, 1], [2, 1], [7, 1], [4, 1], [1, 1], [17, 1], [6, 1], [11, 1], [4, 1], [3, 1], [9, 1], [8, 1], [1, 1], [2, 1], [14, 1], [16, 1], [1, 1], [7, 1], [1, 1], [1, 1], [10, 1], [3, 1], [4, 1], [6, 1], [8, 1], [2, 1], [5, 1], [11, 1], [4, 1], [1, 1], [9, 1], [6, 1], [3, 1], [1, 1], [2, 1], [6, 0], [6, 0], [6, 0], [10, 0], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [6, 1], [4, 1], [8, 1], [6, 1], [11, 1], [7, 1], [3, 1], [34, 1], [16, 1], [17, 1], [5, 1], [4, 1], [7, 1], [6, 1], [16, 1], [5, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [8, 1], [1, 1], [1, 1], [6, 1], [2, 1], [1, 1], [1, 1], [6, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [5, 1], [1, 1], [2, 1], [2, 1], [1, 1], [3, 1], [1, 1], [3, 1], [2, 1], [3, 1], [3, 1], [5, 1], [1, 1], [17, 1], [4, 1], [4, 1], [4, 1], [5, 1], [7, 1], [2, 1], [5, 1], [3, 1], [2, 1], [6, 1], [1, 1], [6, 1], [2, 1], [7, 0], [3, 1], [1, 1], [2, 1], [4, 1], [2, 1], [2, 1], [4, 1], [3, 1], [3, 1], [5, 0], [4, 1], [1, 1], [2, 1], [7, 1], [3, 1], [8, 1], [6, 1], [3, 1], [3, 1], [4, 1], [2, 1], [1, 1], [4, 1], [1, 1], [2, 1], [6, 1], [3, 1], [4, 1], [9, 0], [14, 0], [16, 1], [8, 1], [33, 0], [5, 1], [14, 1], [8, 1], [1, 1], [1, 1], [12, 0], [2, 0], [16, 1], [2, 1], [2, 1], [1, 1], [2, 1], [1, 1], [5, 1], [2, 1], [2, 1], [5, 1], [11, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [5, 1], [2, 1], [3, 1], [2, 1], [1, 1], [1, 1], [9, 1], [3, 1], [8, 1], [4, 1], [6, 1], [7, 1], [2, 1], [1, 1], [2, 1], [2, 1], [2, 1], [9, 1], [2, 1], [3, 1], [3, 1], [6, 1], [2, 1], [6, 1], [2, 1], [1, 1], [4, 1], [1, 1], [4, 1], [8, 1], [2, 1], [5, 1], [4, 1], [55, 1], [8, 1], [1, 1], [5, 1], [7, 1], [5, 1], [11, 1], [14, 1], [4, 1], [4, 1], [2, 1], [1, 1], [17, 1], [2, 1], [8, 1], [4, 1], [14, 1], [23, 1], [5, 1], [18, 1], [5, 1], [7, 1], [1, 1], [6, 1], [1, 1], [10, 1], [1, 1], [20, 1], [24, 1], [6, 1], [53, 1], [37, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 0], [1, 1], [1, 1], [3, 0], [1, 0], [1, 1], [10, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [5, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [2, 1], [2, 1], [1, 1], [3, 1], [17, 1], [4, 1], [3, 1], [5, 1], [9, 1], [12, 1], [1, 1], [13, 1], [6, 1], [8, 1], [3, 1], [5, 1], [18, 1], [4, 1], [5, 1], [6, 1], [3, 1], [2, 1], [1, 1], [2, 1], [25, 1], [2, 1], [2, 1], [2, 1], [1, 1], [8, 1], [1, 1], [4, 1], [2, 1], [1, 1], [2, 1], [4, 1], [5, 1], [52, 1], [6, 1], [46, 1], [6, 1], [8, 1], [6, 1], [24, 0], [2, 1], [15, 1], [14, 1], [16, 1], [42, 1], [8, 1], [4, 1], [2, 1], [3, 1], [2, 1], [6, 1], [9, 1], [5, 1], [10, 1], [1, 1], [2, 1], [2, 1], [5, 1], [2, 1], [1, 1], [4, 1], [7, 1], [3, 1], [3, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [4, 1], [10, 1], [1, 1], [4, 1], [2, 1], [4, 1], [1, 1], [1, 1], [6, 1], [7, 1], [1, 1], [2, 1], [15, 1], [10, 1], [4, 1], [10, 0], [9, 0], [11, 0], [6, 1], [24, 0], [1, 0], [6, 0], [11, 1], [2, 1], [4, 1], [1, 1], [1, 1], [11, 1], [8, 1], [6, 1], [1, 1], [3, 1], [5, 1], [3, 1], [3, 1], [1, 0], [4, 1], [8, 0], [3, 0], [7, 1], [16, 0], [15, 1], [9, 0], [3, 1], [2, 1], [5, 1], [10, 1], [18, 1], [5, 1], [6, 1], [5, 1], [27, 1], [3, 1], [20, 0], [21, 1], [35, 0], [8, 1], [11, 0], [3, 1], [2, 1], [6, 1], [1, 1], [1, 1], [8, 1], [1, 1], [9, 1], [2, 1], [8, 0], [2, 1], [1, 1], [2, 1], [4, 1], [1, 1], [4, 1], [6, 1], [6, 1], [1, 1], [2, 1], [10, 0], [1, 1], [3, 1], [3, 1], [1, 1], [1, 1], [3, 1], [5, 1], [2, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [7, 1], [2, 1], [1, 1], [2, 1], [4, 1], [1, 1], [7, 1], [7, 1], [3, 1], [1, 1], [3, 1], [4, 1], [3, 1], [4, 1], [1, 1], [8, 1], [10, 1], [3, 1], [10, 1], [8, 1], [9, 1], [10, 1], [6, 1], [3, 1], [4, 1], [2, 1], [1, 1], [2, 1], [2, 1], [4, 1], [6, 1], [6, 1], [4, 1], [7, 1], [1, 1], [3, 1], [2, 1], [3, 1], [1, 1], [1, 1], [3, 1], [3, 1], [2, 1], [1, 1], [1, 1], [3, 1], [3, 1], [1, 1], [16, 1], [3, 1], [5, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [3, 1], [2, 1], [9, 1], [2, 1], [4, 1], [11, 1], [21, 1], [7, 1], [14, 1], [7, 1], [7, 1], [3, 1], [2, 1], [26, 1], [18, 1], [20, 1], [11, 1], [1, 1], [1, 1], [2, 1], [1, 1], [19, 1], [2, 1], [1, 1], [8, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [5, 1], [4, 1], [1, 1], [12, 1], [11, 1], [2, 1], [6, 1], [12, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [12, 1], [5, 1], [1, 1], [3, 1], [18, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [6, 1], [1, 1], [8, 1], [1, 1], [3, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [2, 1], [2, 1], [5, 1], [1, 1], [1, 1], [7, 1], [4, 1], [8, 1], [4, 1], [2, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [6, 1], [2, 1], [7, 1], [1, 1], [7, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [12, 1], [7, 1], [5, 1], [8, 0], [11, 1], [4, 1], [3, 1], [1, 0], [1, 1], [9, 1], [1, 1], [2, 1], [6, 1], [9, 0], [12, 0], [2, 1], [3, 1], [6, 1], [4, 1], [4, 1], [4, 1], [2, 1], [1, 1], [12, 1], [3, 1], [14, 1], [10, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [12, 1], [1, 1], [2, 1], [14, 1], [2, 1], [3, 1], [3, 1], [3, 1], [4, 1], [1, 1], [2, 1], [3, 1], [1, 1], [11, 1], [4, 1], [1, 1], [2, 1], [9, 1], [9, 1], [8, 1], [5, 1], [10, 1], [9, 0], [2, 1], [1, 1], [8, 1], [3, 1], [2, 1], [5, 1], [2, 1], [8, 1], [4, 1], [2, 1], [4, 1], [11, 1], [7, 1], [1, 1], [8, 1], [6, 1], [17, 1], [6, 1], [4, 0], [5, 1], [8, 1], [9, 0], [5, 1], [8, 1], [37, 0], [13, 0], [4, 0], [10, 0], [5, 0], [11, 0], [3, 1], [3, 1], [1, 1], [6, 1], [2, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [4, 1], [16, 1], [12, 1], [7, 0], [5, 1], [6, 0], [12, 0], [2, 1], [14, 1], [10, 1], [7, 1], [2, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [7, 1], [12, 1], [6, 1], [8, 1], [21, 0], [7, 1], [23, 1], [10, 1], [2, 1], [3, 1], [6, 1], [2, 1], [12, 1], [13, 1], [15, 1], [4, 1], [1, 1], [21, 1], [6, 1], [8, 1], [39, 1], [1, 1], [4, 1], [9, 1], [1, 1], [1, 1], [1, 1], [4, 1], [4, 1], [1, 1], [3, 1], [1, 1], [2, 1], [3, 1], [6, 0], [1, 1], [3, 0], [2, 1], [1, 1], [9, 1], [3, 1], [1, 1], [4, 1], [3, 0], [1, 1], [2, 1], [3, 1], [3, 1], [1, 1], [2, 1], [5, 1], [1, 1], [1, 1], [8, 1], [6, 1], [1, 1], [1, 1], [1, 1], [10, 1], [1, 1], [7, 1], [4, 1], [9, 1], [6, 1], [5, 1], [2, 1], [2, 1], [9, 1], [8, 1], [11, 1], [1, 1], [2, 1], [12, 1], [1, 1], [3, 1], [2, 1], [3, 1], [5, 1], [4, 1], [4, 1], [1, 1], [2, 1], [3, 1], [15, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 1], [8, 1], [2, 1], [8, 1], [7, 1], [3, 1], [3, 1], [22, 1], [1, 1], [4, 1], [7, 1], [4, 1], [3, 1], [4, 1], [1, 1], [1, 1], [16, 1], [1, 1], [3, 1], [7, 1], [1, 1], [6, 1], [16, 1], [3, 1], [2, 1], [6, 1], [2, 1], [4, 1], [11, 1], [8, 1], [2, 1], [3, 1], [31, 1], [5, 1], [12, 1], [2, 1], [5, 1], [7, 1], [1, 1], [12, 1], [3, 1], [1, 1], [1, 1], [7, 1], [13, 1], [4, 1], [6, 1], [21, 1], [3, 1], [1, 1], [6, 1], [2, 1], [3, 1], [2, 1], [3, 1], [2, 1], [4, 1], [3, 1], [2, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [4, 1], [1, 1], [7, 1], [9, 1], [22, 1], [2, 1], [12, 0], [9, 1], [3, 1], [20, 1], [14, 1], [22, 1], [21, 1], [2, 1], [8, 1], [7, 1], [2, 1], [1, 1], [1, 1], [1, 1], [6, 1], [3, 1], [4, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [19, 1], [2, 1], [1, 1], [6, 1], [23, 1], [2, 1], [8, 1], [5, 1], [1, 1], [8, 1], [16, 1], [20, 1], [13, 1], [15, 1], [12, 1], [8, 1], [1, 1], [2, 1], [2, 1], [5, 1], [2, 1], [7, 1], [3, 1], [3, 1], [1, 1], [9, 1], [9, 0], [6, 0], [4, 1], [1, 1], [7, 1], [11, 1], [1, 1], [4, 1], [6, 1], [6, 1], [2, 1], [5, 1], [3, 1], [6, 0], [9, 0], [1, 0], [6, 1], [7, 0], [11, 1], [3, 1], [6, 0], [20, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [3, 1], [3, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [5, 1], [13, 1], [8, 1], [26, 1], [1, 1], [3, 1], [7, 1], [1, 1], [6, 1], [7, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [3, 1], [4, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [2, 1], [1, 1], [11, 1], [7, 1], [3, 1], [8, 1], [13, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [3, 1], [1, 1], [4, 1], [3, 1], [11, 1], [18, 1], [41, 1], [4, 1], [4, 1], [6, 1], [2, 1], [1, 1], [8, 1], [4, 1], [1, 1], [19, 1], [20, 1], [21, 1], [11, 1], [10, 1], [35, 0], [71, 1], [3, 1], [10, 0], [11, 0], [32, 0], [8, 1], [4, 1], [55, 1], [2, 1], [3, 1], [8, 1], [6, 1], [4, 1], [2, 1], [1, 1], [13, 1], [5, 1], [1, 1], [3, 1], [1, 1], [1, 1], [5, 1], [3, 1], [2, 1], [8, 1], [3, 1], [3, 1], [6, 1], [7, 1], [10, 1], [19, 1], [5, 1], [7, 0], [5, 1], [9, 1], [4, 1], [1, 1], [15, 0], [5, 1], [21, 1], [2, 1], [2, 1], [2, 1], [3, 1], [1, 1], [11, 1], [9, 1], [12, 1], [3, 1], [4, 1], [2, 1], [2, 1], [3, 1], [17, 1], [9, 1], [9, 1], [49, 1], [5, 1], [10, 1], [93, 1], [19, 1], [10, 1], [14, 1], [28, 1], [51, 1], [39, 1], [11, 1], [3, 1], [13, 1], [19, 1], [21, 1], [3, 1], [2, 1], [2, 1], [1, 1], [23, 1], [1, 1], [4, 1], [2, 1], [7, 1], [3, 1], [3, 1], [10, 1], [3, 1], [2, 1], [3, 1], [2, 1], [6, 1], [2, 1], [11, 1], [6, 1], [13, 1], [4, 1], [2, 1], [2, 1], [5, 1], [6, 1], [13, 1], [2, 1], [9, 1], [21, 1], [14, 0], [3, 1], [1, 0], [6, 1], [2, 1], [2, 1], [2, 1], [4, 0], [3, 0], [4, 0], [2, 1], [2, 1], [11, 1], [1, 1], [10, 1], [2, 1], [1, 1], [2, 1], [4, 1], [9, 1], [1, 1], [4, 1], [1, 1], [4, 1], [10, 1], [7, 1], [1, 1], [5, 1], [10, 1], [25, 1], [7, 1], [5, 1], [8, 1], [4, 1], [1, 1], [1, 1], [4, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [9, 0], [2, 1], [3, 0], [4, 1], [1, 1], [1, 1], [11, 1], [1, 1], [1, 1], [4, 1], [1, 1], [3, 1], [6, 1], [2, 1], [13, 1], [6, 1], [3, 0], [10, 0], [8, 1], [28, 1], [10, 1], [13, 0], [3, 0], [3, 0], [19, 0], [10, 0], [9, 1], [17, 1], [3, 1], [2, 1], [2, 1], [1, 1], [8, 1], [4, 1], [1, 1], [2, 1], [5, 1], [1, 1], [3, 1], [6, 1], [1, 1], [1, 1], [4, 1], [2, 1], [2, 1], [7, 1], [9, 0], [18, 0], [4, 1], [27, 0], [6, 1], [9, 0], [1, 1], [2, 1], [11, 1], [17, 1], [12, 0], [7, 1], [3, 1], [6, 1], [7, 1], [2, 1], [2, 1], [1, 1], [1, 1], [3, 1], [1, 1], [5, 1], [3, 1], [4, 1], [6, 1], [1, 1], [6, 1], [1, 1], [5, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [5, 1], [3, 1], [1, 1], [5, 1], [1, 1], [1, 1], [5, 1], [1, 1], [3, 1], [1, 1], [3, 1], [1, 1], [2, 1], [2, 1], [9, 0], [1, 1], [7, 1], [1, 1], [2, 1], [3, 1], [1, 1], [3, 1], [1, 1], [19, 1], [2, 1], [3, 1], [4, 1], [17, 1], [7, 1], [1, 1], [5, 1], [1, 1], [6, 1], [15, 1], [6, 1], [7, 1], [7, 1], [8, 1], [6, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [7, 1], [4, 1], [1, 1], [6, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [22, 1], [6, 1], [1, 1], [5, 1], [1, 1], [9, 1], [1, 1], [1, 1], [2, 1], [3, 1], [2, 1], [13, 1], [4, 1], [3, 1], [3, 1], [12, 1], [1, 1], [14, 1], [6, 1], [12, 1], [2, 1], [7, 1], [12, 1], [5, 1], [9, 1], [10, 1], [2, 1], [6, 1], [10, 1], [1, 1], [18, 1], [6, 1], [2, 1], [3, 1], [3, 1], [1, 1], [4, 1], [2, 1], [1, 1], [2, 1], [7, 1], [14, 1], [2, 1], [14, 1], [19, 1], [28, 1], [2, 1], [6, 1], [21, 1], [5, 1], [7, 1], [8, 1], [2, 1], [43, 1], [35, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [3, 1], [2, 1], [3, 1], [4, 1], [6, 1], [3, 1], [19, 0], [2, 1], [4, 1], [2, 1], [2, 1], [7, 1], [3, 1], [5, 1], [2, 1], [2, 1], [2, 1], [1, 1], [5, 1], [5, 1], [2, 1], [3, 1], [1, 1], [6, 1], [19, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [13, 1], [5, 0], [1, 1], [2, 1], [1, 1], [2, 1], [3, 1], [3, 1], [6, 1], [3, 1], [5, 1], [7, 0], [4, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [4, 1], [1, 1], [4, 1], [11, 1], [1, 1], [1, 1], [2, 1], [4, 1], [7, 1], [5, 1], [3, 1], [12, 1], [2, 1], [2, 1], [11, 1], [5, 1], [36, 1], [16, 1], [10, 1], [24, 0], [12, 1], [42, 1], [3, 1], [1, 1], [7, 1], [13, 0], [9, 0], [10, 1], [3, 1], [5, 1], [1, 1], [10, 1], [7, 1], [8, 1], [1, 1], [2, 1], [3, 1], [2, 1], [1, 1], [1, 1], [5, 1], [1, 1], [8, 1], [3, 1], [1, 1], [2, 1], [1, 0], [2, 1], [3, 1], [1, 1], [2, 1], [1, 1], [6, 0], [3, 1], [8, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 1], [5, 1], [1, 1], [10, 1], [2, 1], [5, 1], [4, 1], [6, 1], [12, 1], [16, 0], [5, 1], [11, 0], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [4, 1], [1, 1], [1, 1], [4, 1], [37, 1], [1, 1], [1, 1], [18, 1], [3, 1], [4, 1], [1, 1], [1, 1], [2, 1], [9, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [40, 1], [5, 1], [1, 1], [1, 1], [2, 1], [1, 1], [5, 1], [14, 1], [15, 1], [14, 1], [3, 1], [4, 1], [1, 0], [4, 1], [5, 1], [11, 1], [2, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [5, 1], [2, 1], [3, 1], [9, 0], [15, 0], [3, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [9, 1], [3, 1], [10, 1], [4, 1], [2, 1], [1, 1], [4, 1], [1, 1], [16, 1], [1, 1], [1, 1], [4, 1], [4, 1], [4, 1], [1, 1], [3, 1], [1, 1], [1, 1], [8, 1], [3, 1], [1, 1], [1, 1], [4, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [5, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 0], [1, 1], [3, 0], [7, 1], [8, 0], [10, 0], [1, 1], [5, 0], [2, 0], [2, 1], [5, 1], [1, 1], [4, 1], [2, 1], [1, 1], [9, 1], [7, 1], [8, 1], [3, 1], [4, 1], [7, 1], [1, 1], [4, 1], [6, 1], [3, 1], [1, 1], [6, 1], [11, 1], [3, 1], [1, 1], [2, 1], [2, 1], [15, 1], [2, 1], [4, 1], [1, 1], [1, 1], [5, 1], [11, 1], [2, 1], [5, 1], [18, 1], [3, 1], [3, 1], [2, 1], [1, 1], [10, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [6, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 0], [3, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [10, 0], [3, 1], [2, 1], [7, 0], [3, 1], [1, 1], [1, 1], [3, 1], [3, 1], [2, 1], [10, 1], [6, 0], [2, 0], [3, 1], [9, 0], [6, 0], [4, 1], [1, 1], [4, 1], [3, 1], [1, 1], [3, 1], [12, 0], [1, 1], [9, 0], [7, 0], [6, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [6, 1], [1, 1], [3, 1], [3, 1], [9, 1], [8, 0], [12, 0], [8, 0], [3, 1], [13, 0], [9, 1], [7, 0], [3, 0], [10, 1], [6, 0], [5, 0], [1, 0], [1, 0], [1, 0], [2, 0], [1, 0], [3, 0], [1, 0], [1, 1], [2, 1], [2, 1], [1, 1], [7, 1], [5, 1], [1, 1], [1, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [2, 1], [6, 1], [1, 1], [1, 1], [4, 1], [4, 1], [1, 1], [1, 1], [3, 1], [3, 1], [2, 1], [3, 1], [1, 1], [2, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [5, 1], [2, 1], [4, 1], [14, 1], [15, 1], [1, 1], [2, 1], [7, 1], [3, 1], [1, 1], [3, 1], [13, 1], [7, 1], [8, 1], [3, 1], [1, 1], [1, 1], [4, 1], [2, 1], [2, 1], [4, 1], [7, 1], [3, 1], [2, 1], [1, 1], [3, 1], [4, 1], [5, 1], [1, 1], [3, 1], [1, 1], [1, 1], [2, 1], [11, 1], [3, 1], [2, 1], [10, 1], [5, 1], [6, 1], [3, 1], [1, 1], [14, 1], [3, 1], [13, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [15, 1], [8, 1], [4, 0], [2, 1], [2, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [6, 1], [6, 1], [12, 1], [5, 1], [5, 0], [10, 1], [1, 1], [13, 0], [6, 0], [4, 0], [5, 0], [2, 1], [6, 0], [9, 1], [7, 1], [10, 1], [1, 1], [12, 1], [8, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [2, 1], [2, 1], [3, 1], [3, 1], [3, 1], [2, 1], [9, 1], [3, 1], [1, 1], [9, 1], [14, 1], [4, 1], [12, 1], [4, 1], [19, 1], [5, 1], [2, 1], [12, 1], [3, 1], [7, 1], [5, 1], [1, 1], [6, 1], [11, 1], [2, 1], [4, 1], [2, 1], [1, 1], [3, 1], [1, 1], [6, 1], [1, 1], [4, 1], [2, 1], [9, 1], [13, 1], [3, 1], [7, 1], [1, 1], [7, 1], [26, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [2, 1], [1, 1], [3, 1], [2, 1], [2, 1], [5, 1], [4, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [4, 1], [2, 1], [4, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [2, 1], [3, 0], [2, 1], [2, 1], [9, 0], [3, 0], [2, 0], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [6, 0], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [4, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 1], [4, 1], [7, 1], [3, 1], [1, 1], [5, 1], [10, 1], [6, 1], [16, 1], [2, 1], [3, 1], [5, 1], [7, 1], [8, 1], [4, 1], [7, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [19, 1], [9, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [3, 1], [2, 1], [8, 1], [21, 1], [9, 1], [3, 1], [8, 1], [9, 1], [1, 1], [5, 1], [2, 1], [4, 1], [9, 1], [7, 1], [8, 1], [6, 1], [11, 1], [33, 1], [9, 1], [21, 1], [9, 1], [2, 1], [6, 1], [7, 1], [60, 1], [10, 1], [38, 1], [5, 1], [2, 1], [1, 1], [1, 1], [2, 1], [7, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [9, 1], [2, 1], [1, 1], [1, 1], [7, 0], [1, 1], [5, 0], [4, 1], [1, 1], [7, 1], [1, 1], [4, 1], [9, 0], [13, 1], [14, 1], [6, 0], [12, 0], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [2, 1], [5, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [8, 1], [18, 0], [5, 1], [6, 1], [5, 1], [27, 0], [3, 1], [14, 0], [11, 1], [2, 1], [3, 1], [26, 1], [5, 1], [6, 1], [15, 1], [1, 1], [5, 1], [1, 1], [1, 1], [1, 1], [10, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [11, 1], [1, 1], [2, 1], [25, 1], [17, 1], [2, 1], [1, 1], [8, 1], [8, 1], [9, 1], [14, 1], [10, 1], [3, 1], [1, 1], [8, 1], [14, 1], [34, 1], [21, 0], [13, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [4, 1], [5, 0], [8, 1], [1, 1], [3, 1], [35, 0], [3, 1], [3, 1], [7, 0], [3, 1], [7, 0], [2, 1], [1, 1], [2, 0], [2, 1], [2, 1], [4, 0], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [9, 1], [1, 1], [1, 1], [8, 1], [18, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [4, 1], [2, 1], [1, 1], [7, 1], [9, 1], [3, 1], [1, 1], [4, 1], [3, 1], [1, 1], [4, 1], [1, 1], [2, 1], [5, 1], [6, 1], [6, 1], [1, 1], [8, 1], [5, 1], [3, 1], [5, 1], [15, 1], [5, 1], [2, 1], [7, 1], [4, 1], [3, 1], [3, 1], [7, 1], [1, 1], [1, 1], [1, 1], [1, 1], [6, 1], [5, 1], [1, 0], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [2, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [6, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [2, 1], [14, 1], [7, 1], [5, 1], [5, 1], [24, 1], [1, 1], [5, 1], [7, 1], [1, 1], [1, 1], [5, 1], [3, 1], [6, 1], [8, 1], [1, 1], [1, 1], [2, 1], [6, 1], [5, 1], [4, 1], [2, 1], [2, 1], [4, 1], [3, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [8, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [2, 1], [1, 1], [2, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [3, 0], [3, 1], [10, 1], [36, 1], [3, 1], [8, 0], [3, 1], [9, 0], [8, 0], [1, 0], [5, 0], [6, 1], [14, 0], [7, 1], [1, 1], [8, 0], [18, 0], [6, 1], [12, 1], [14, 0], [21, 0], [5, 1], [2, 1], [1, 1], [2, 1], [1, 1], [4, 1], [6, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [4, 1], [1, 1], [5, 1], [4, 1], [1, 1], [2, 1], [4, 1], [13, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [6, 1], [4, 1], [4, 1], [2, 1], [8, 1], [3, 1], [6, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [5, 0], [6, 0], [2, 1], [4, 1], [15, 1], [8, 1], [10, 1], [5, 1], [1, 1], [3, 1], [3, 1], [4, 1], [5, 0], [6, 1], [7, 1], [1, 1], [9, 1], [1, 1], [1, 1], [1, 1], [7, 1], [4, 1], [2, 1], [6, 1], [1, 1], [1, 1], [1, 1], [2, 1], [5, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [4, 1], [3, 0], [2, 1], [2, 1], [20, 1], [29, 1], [1, 1], [8, 1], [6, 1], [17, 1], [1, 1], [2, 1], [1, 1], [6, 1], [20, 1], [2, 1], [8, 1], [5, 1], [4, 0], [17, 0], [15, 1], [3, 0], [7, 0], [6, 0], [1, 1], [31, 0], [11, 0], [7, 0], [3, 0], [6, 1], [6, 1], [2, 1], [3, 1], [10, 1], [9, 1], [3, 1], [3, 1], [9, 1], [3, 1], [2, 1], [4, 1], [1, 1], [7, 1], [26, 1], [3, 1], [5, 1], [3, 1], [4, 0], [3, 1], [3, 1], [3, 1], [4, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [2, 1], [2, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [3, 1], [1, 1], [1, 1], [3, 1], [32, 1], [1, 1], [2, 1], [21, 1], [51, 1], [9, 1], [7, 1], [8, 1], [18, 1], [13, 1], [10, 1], [1, 1], [5, 1], [6, 1], [9, 0], [3, 1], [2, 1], [3, 0], [16, 0], [3, 1], [8, 1], [6, 1], [4, 0], [7, 0], [14, 0], [8, 1], [9, 1], [12, 1], [6, 0], [22, 0], [18, 1], [6, 1], [3, 1], [10, 1], [10, 1], [14, 1], [35, 1], [12, 1], [4, 1], [58, 1], [2, 1], [4, 1], [11, 1], [30, 1], [11, 1], [2, 1], [13, 1], [5, 1], [30, 1], [5, 1], [1, 1], [6, 1], [7, 1], [3, 1], [11, 1], [12, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [10, 1], [1, 1], [1, 1], [3, 1], [5, 1], [12, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [8, 1], [1, 1], [1, 1], [12, 1], [5, 1], [1, 1], [44, 1], [3, 1], [8, 1], [1, 1], [5, 1], [4, 1], [9, 1], [5, 1], [2, 1], [5, 1], [4, 1], [35, 1], [2, 1], [19, 1], [67, 0], [26, 1], [11, 1], [66, 0], [14, 1], [24, 1], [11, 1], [2, 1], [11, 1], [15, 1], [14, 1], [10, 1], [2, 1], [5, 1], [6, 1], [2, 1], [4, 1], [20, 1], [1, 1], [5, 1], [25, 1], [10, 1], [1, 1], [8, 1], [4, 1], [12, 1], [5, 1], [3, 1], [23, 1], [2, 1], [7, 1], [11, 0], [4, 1], [5, 1], [5, 1], [1, 1], [19, 0], [1, 1], [10, 1], [12, 1], [3, 1], [2, 1], [2, 1], [1, 1], [8, 1], [1, 1], [9, 1], [5, 1], [5, 1], [2, 1], [9, 1], [3, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [2, 1], [2, 1], [5, 1], [1, 1], [4, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [15, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [6, 1], [3, 1], [4, 1], [2, 1], [1, 1], [1, 0], [1, 1], [10, 1], [2, 1], [10, 0], [4, 1], [5, 1], [7, 0], [5, 1], [4, 1], [8, 1], [3, 1], [3, 1], [2, 1], [1, 1], [4, 1], [3, 1], [1, 1], [1, 1], [3, 1], [2, 1], [7, 1], [5, 1], [8, 1], [1, 1], [39, 1], [16, 1], [2, 1], [3, 1], [16, 1], [13, 1], [1, 1], [6, 1], [2, 1], [2, 1], [5, 1], [3, 1], [3, 1], [11, 1], [9, 1], [1, 1], [1, 1], [3, 1], [4, 1], [2, 1], [12, 1], [4, 1], [4, 1], [2, 1], [6, 1], [2, 1], [1, 1], [1, 1], [3, 1], [2, 1], [2, 1], [11, 1], [2, 1], [21, 1], [16, 1], [6, 1], [2, 1], [2, 1], [1, 1], [8, 1], [1, 1], [1, 1], [2, 0], [12, 0], [19, 0], [1, 1], [11, 0], [8, 0], [3, 1], [3, 1], [3, 1], [15, 1], [5, 1], [5, 1], [4, 1], [2, 1], [6, 1], [13, 0], [2, 0], [2, 1], [8, 1], [4, 0], [6, 0], [3, 1], [2, 1], [1, 1], [1, 1], [5, 1], [2, 1], [5, 1], [1, 1], [3, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [4, 1], [2, 1], [4, 1], [11, 1], [19, 1], [2, 1], [27, 1], [4, 1], [3, 1], [13, 1], [13, 1], [15, 1], [7, 1], [8, 0], [9, 1], [1, 1], [3, 1], [1, 1], [6, 1], [4, 1], [4, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [5, 1], [1, 1], [1, 1], [5, 1], [4, 1], [3, 1], [2, 1], [3, 1], [1, 1], [1, 1], [2, 1], [3, 1], [8, 1], [5, 1], [2, 1], [2, 1], [1, 1], [4, 0], [8, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [7, 0], [5, 1], [6, 1], [4, 1], [1, 1], [2, 1], [9, 1], [3, 1], [12, 1], [5, 1], [4, 1], [3, 1], [1, 1], [11, 1], [1, 1], [1, 1], [1, 1], [1, 1], [10, 1], [3, 1], [2, 1], [2, 1], [9, 1], [26, 1], [2, 1], [11, 1], [1, 1], [7, 1], [4, 1], [10, 1], [25, 1], [40, 1], [25, 1], [6, 1], [2, 1], [3, 1], [2, 1], [1, 1], [5, 1], [16, 1], [1, 1], [12, 1], [4, 1], [6, 1], [12, 1], [10, 1], [6, 1], [15, 0], [2, 1], [8, 1], [21, 1], [3, 1], [10, 1], [21, 1], [3, 1], [8, 1], [6, 1], [1, 1], [8, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [2, 1], [3, 1], [7, 1], [2, 1], [1, 1], [2, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [3, 1], [1, 1], [6, 1], [8, 1], [17, 1], [2, 1], [3, 1], [1, 1], [14, 1], [8, 1], [3, 1], [9, 1], [32, 1], [7, 1], [7, 1], [3, 1], [2, 1], [43, 1], [5, 1], [2, 1], [26, 1], [3, 1], [3, 1], [3, 1], [30, 1], [14, 1], [3, 1], [5, 1], [7, 1], [6, 1], [12, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [2, 1], [1, 1], [5, 1], [1, 1], [1, 1], [1, 1], [8, 1], [2, 1], [5, 1], [10, 1], [5, 1], [3, 1], [2, 1], [1, 0], [9, 1], [3, 1], [18, 1], [4, 1], [3, 1], [1, 1], [3, 1], [9, 1], [1, 1], [2, 1], [20, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [3, 1], [1, 1], [3, 1], [7, 1], [1, 1], [2, 1], [2, 1], [4, 1], [1, 1], [6, 1], [6, 1], [8, 1], [6, 1], [2, 1], [7, 1], [2, 1], [3, 1], [5, 1], [3, 1], [5, 1], [1, 1], [6, 1], [7, 1], [3, 1], [20, 1], [12, 1], [16, 1], [11, 1], [9, 1], [4, 1], [8, 1], [26, 1], [29, 1], [12, 1], [4, 1], [7, 1], [9, 1], [6, 1], [2, 1], [2, 0], [4, 1], [1, 1], [29, 0], [44, 0], [2, 1], [7, 1], [3, 1], [3, 1], [1, 1], [4, 1], [1, 1], [8, 1], [9, 0], [13, 1], [21, 1], [8, 1], [26, 0], [17, 1], [16, 1], [10, 1], [3, 1], [1, 0], [9, 1], [6, 1], [4, 1], [15, 1], [11, 1], [13, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [7, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [5, 0], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [6, 1], [1, 1], [2, 1], [4, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [6, 1], [3, 1], [1, 1], [1, 1], [9, 1], [1, 1], [5, 1], [2, 1], [13, 0], [1, 1], [5, 1], [9, 1], [2, 1], [6, 1], [1, 1], [2, 1], [2, 1], [2, 1], [2, 1], [4, 1], [14, 1], [3, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [10, 1], [1, 1], [1, 1], [2, 1], [1, 0], [1, 1], [1, 1], [7, 1], [1, 1], [2, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [28, 1], [6, 1], [7, 1], [2, 1], [5, 1], [2, 1], [10, 1], [6, 1], [7, 1], [1, 1], [11, 1], [5, 1], [1, 1], [6, 1], [10, 1], [12, 1], [1, 1], [2, 1], [4, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [2, 1], [1, 1], [6, 1], [1, 1], [2, 1], [4, 1], [3, 1], [9, 1], [1, 1], [6, 1], [1, 1], [17, 1], [4, 1], [1, 1], [1, 1], [2, 1], [5, 1], [9, 1], [4, 1], [1, 1], [3, 1], [1, 1], [6, 1], [4, 1], [7, 1], [1, 1], [1, 1], [4, 1], [2, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [2, 1], [3, 1], [5, 1], [3, 1], [2, 1], [6, 1], [2, 1], [4, 1], [5, 1], [3, 1], [2, 1], [1, 1], [6, 1], [5, 1], [8, 1], [6, 0], [3, 1], [1, 1], [4, 1], [2, 1], [2, 1], [3, 0], [2, 1], [9, 1], [2, 1], [2, 1], [4, 1], [1, 1], [35, 1], [5, 1], [3, 1], [2, 1], [2, 1], [3, 1], [2, 1], [2, 1], [1, 1], [4, 1], [16, 1], [8, 1], [14, 1], [3, 1], [6, 1], [12, 1], [7, 1], [61, 0], [31, 1], [67, 0], [49, 1], [6, 1], [7, 1], [4, 1], [24, 1], [16, 1], [3, 1], [3, 0], [3, 1], [1, 1], [8, 0], [27, 0], [1, 1], [3, 1], [3, 1], [7, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [2, 1], [3, 1], [9, 0], [1, 1], [15, 1], [1, 1], [3, 1], [12, 0], [4, 0], [3, 1], [3, 1], [7, 0], [10, 1], [1, 1], [3, 1], [1, 1], [3, 1], [7, 1], [9, 1], [5, 1], [2, 1], [14, 1], [17, 0], [3, 1], [15, 1], [18, 1], [11, 0], [24, 1], [10, 1], [13, 1], [6, 1], [8, 1], [14, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [10, 1], [1, 1], [1, 1], [2, 1], [1, 1], [24, 1], [3, 1], [12, 1], [4, 1], [6, 1], [24, 1], [22, 1], [9, 1], [14, 0], [27, 1], [4, 0], [6, 1], [3, 1], [8, 1], [27, 1], [3, 1], [3, 1], [1, 1], [3, 1], [2, 1], [3, 1], [10, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [6, 1], [1, 1], [1, 1], [8, 1], [6, 1], [6, 1], [5, 1], [2, 1], [9, 1], [3, 1], [1, 1], [4, 1], [4, 1], [3, 1], [4, 1], [10, 1], [1, 1], [10, 1], [2, 1], [1, 1], [5, 1], [7, 1], [5, 1], [5, 1], [2, 1], [6, 1], [7, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [1, 1], [6, 1], [1, 1], [4, 1], [1, 1], [1, 1], [6, 1], [5, 1], [3, 1], [9, 1], [1, 1], [1, 1], [2, 1], [5, 1], [3, 1], [4, 1], [7, 1], [7, 1], [3, 1], [3, 1], [6, 1], [8, 1], [1, 0], [8, 0], [3, 1], [8, 1], [2, 1], [7, 1], [12, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 0], [1, 1], [1, 1], [5, 1], [10, 1], [10, 1], [27, 1], [4, 1], [2, 1], [2, 1], [2, 1], [4, 1], [1, 1], [3, 1], [2, 1], [4, 1], [3, 1], [1, 1], [4, 1], [6, 1], [10, 1], [23, 1], [1, 1], [12, 1], [3, 1], [23, 1], [5, 1], [6, 1], [14, 1], [16, 1], [15, 1], [13, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [3, 1], [5, 1], [3, 1], [2, 1], [1, 1], [7, 1], [6, 1], [1, 1], [7, 1], [48, 1], [1, 1], [9, 1], [3, 1], [3, 1], [3, 1], [2, 1], [10, 1], [3, 1], [6, 1], [1, 1], [6, 1], [2, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [5, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [14, 1], [1, 1], [3, 1], [3, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [5, 1], [2, 1], [1, 1], [3, 1], [1, 1], [6, 1], [4, 1], [1, 1], [11, 1], [12, 1], [6, 1], [3, 1], [2, 1], [3, 0], [1, 1], [7, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [5, 1], [2, 1], [3, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [13, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [12, 0], [14, 0], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [3, 0], [3, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [2, 1], [3, 1], [2, 1], [3, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [3, 1], [3, 1], [1, 1], [13, 1], [1, 1], [7, 1], [10, 1], [6, 1], [2, 1], [2, 1], [2, 1], [1, 1], [2, 1], [2, 1], [2, 1], [2, 1], [1, 1], [4, 1], [2, 1], [6, 1], [2, 1], [4, 1], [4, 1], [15, 1], [6, 1], [3, 1], [5, 0], [13, 1], [5, 1], [22, 0], [1, 1], [7, 1], [2, 1], [3, 1], [3, 1], [9, 1], [5, 1], [4, 0], [2, 1], [1, 1], [1, 1], [1, 1], [4, 1], [3, 1], [3, 1], [2, 1], [3, 1], [4, 1], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [6, 1], [2, 1], [9, 1], [4, 1], [22, 1], [2, 1], [4, 0], [1, 1], [3, 0], [2, 1], [1, 1], [2, 1], [6, 1], [4, 1], [15, 0], [3, 1], [2, 1], [2, 1], [3, 1], [8, 1], [4, 1], [3, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [6, 0], [1, 1], [1, 1], [7, 1], [1, 1], [2, 1], [2, 1], [4, 1], [1, 1], [1, 1], [1, 1], [8, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [12, 1], [1, 1], [10, 1], [3, 1], [9, 1], [13, 0], [6, 1], [8, 1], [5, 1], [1, 1], [7, 1], [4, 1], [1, 1], [3, 1], [4, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [4, 1], [1, 1], [9, 1], [2, 1], [2, 1], [7, 1], [58, 0], [2, 1], [11, 1], [1, 1], [4, 1], [1, 1], [4, 1], [10, 1], [4, 1], [28, 1], [3, 1], [1, 1], [1, 1], [2, 1], [1, 1], [3, 1], [5, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 0], [1, 1], [1, 1], [1, 1], [1, 1], [6, 0], [5, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [4, 1], [4, 1], [5, 1], [17, 1], [1, 1], [3, 1], [9, 1], [5, 1], [1, 1], [7, 1], [4, 1], [4, 1], [5, 1], [16, 1], [30, 1], [9, 1], [7, 1], [40, 1], [9, 1], [6, 1], [4, 1], [34, 1], [16, 1], [6, 1], [16, 1], [14, 1], [21, 1], [26, 1], [7, 1], [19, 0], [18, 1], [9, 1], [23, 0], [24, 0], [23, 1], [1, 1], [4, 1], [6, 1], [1, 1], [3, 1], [1, 1], [11, 0], [10, 0], [3, 0], [2, 0], [12, 0], [1, 0], [7, 0], [1, 0], [1, 0], [2, 0], [3, 0], [1, 0], [7, 0], [1, 0], [1, 0], [2, 1], [1, 1], [1, 1], [1, 1], [9, 1], [3, 1], [3, 1], [1, 1], [2, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [2, 1], [1, 0], [5, 0], [7, 1], [2, 0], [4, 0], [5, 0], [2, 1], [3, 0], [4, 1], [7, 1], [3, 0], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [5, 1], [2, 1], [1, 1], [1, 1], [3, 1], [1, 1], [2, 1], [3, 1], [2, 1], [2, 1], [2, 1], [2, 1], [1, 1], [8, 1], [19, 0], [2, 1], [34, 0], [4, 1], [12, 0], [3, 1], [2, 1], [11, 1], [8, 0], [2, 0], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [15, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [8, 0], [2, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 0], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [3, 1], [3, 1], [2, 1], [1, 1], [3, 1], [1, 1], [2, 1], [2, 1], [2, 1], [2, 1], [11, 1], [9, 0], [16, 1], [5, 1], [17, 1], [23, 1], [7, 1], [24, 1], [9, 1], [7, 1], [2, 1], [6, 1], [21, 1], [5, 1], [21, 1], [2, 1], [2, 1], [4, 1], [2, 1], [1, 1], [4, 1], [3, 1], [1, 1], [2, 1], [9, 1], [1, 1], [1, 1], [6, 1], [1, 1], [2, 1], [5, 0], [8, 0], [6, 1], [2, 1], [7, 0], [3, 0], [5, 1], [12, 0], [7, 0], [4, 0], [2, 0], [4, 0], [1, 1], [7, 0], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [3, 1], [1, 1], [8, 1], [7, 1], [2, 1], [27, 1], [7, 1], [7, 0], [1, 0], [6, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 0], [1, 1], [2, 0], [2, 1], [1, 1], [14, 1], [2, 1], [2, 1], [8, 1], [6, 1], [2, 1], [3, 1], [5, 1], [4, 1], [1, 1], [6, 1], [2, 1], [1, 1], [1, 1], [11, 1], [10, 1], [5, 1], [5, 1], [5, 1], [12, 0], [30, 0], [5, 0], [9, 0], [23, 1], [7, 1], [27, 0], [1, 0], [14, 1], [18, 0], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [2, 1], [3, 1], [1, 1], [5, 1], [1, 1], [2, 1], [2, 1], [1, 1], [2, 1], [3, 1], [12, 1], [20, 1], [6, 1], [20, 1], [12, 1], [3, 1], [3, 1], [3, 1], [7, 1], [4, 1], [18, 1], [11, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 0], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [3, 0], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [3, 1], [3, 1], [1, 1], [1, 1], [12, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 1], [6, 1], [5, 1], [4, 1], [17, 1], [19, 1], [4, 1], [5, 1], [1, 1], [7, 1], [5, 1], [2, 1], [15, 1], [8, 1], [9, 1], [11, 1], [3, 1], [6, 1], [1, 1], [1, 1], [5, 1], [5, 1], [5, 1], [1, 1], [8, 1], [11, 1], [3, 1], [4, 1], [1, 1], [6, 1], [20, 1], [6, 0], [4, 1], [4, 1], [2, 0], [3, 1], [6, 1], [9, 0], [13, 1], [8, 1], [7, 1], [3, 1], [6, 0], [5, 0], [9, 0], [8, 1], [2, 1], [9, 0], [4, 0], [2, 1], [32, 1], [28, 1], [3, 1], [14, 0], [8, 1], [5, 1], [6, 1], [2, 1], [4, 1], [11, 0], [4, 1], [8, 0], [6, 1], [1, 1], [1, 1], [10, 1], [1, 1], [5, 0], [2, 0], [3, 1], [5, 1], [5, 1], [26, 1], [3, 0], [6, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 0], [1, 1], [10, 1], [1, 1], [2, 1], [1, 1], [10, 1], [27, 1], [1, 1], [10, 1], [2, 1], [4, 1], [1, 1], [2, 1], [1, 1], [5, 1], [4, 1], [4, 1], [5, 1], [3, 1], [7, 1], [5, 1], [1, 1], [5, 1], [5, 1], [6, 0], [8, 1], [10, 1], [19, 0], [1, 1], [11, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [9, 1], [2, 1], [4, 1], [2, 1], [6, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [1, 1], [3, 1], [15, 1], [2, 1], [2, 1], [4, 1], [10, 1], [13, 1], [6, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 0], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [1, 1], [2, 1], [3, 1], [2, 1], [1, 1], [1, 1], [7, 1], [1, 1], [1, 1], [2, 1], [5, 1], [8, 1], [5, 1], [28, 1], [1, 1], [5, 1], [3, 1], [1, 1], [3, 1], [3, 1], [22, 1], [1, 1], [19, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [21, 1], [9, 1], [2, 1], [1, 1], [4, 1], [6, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [3, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [16, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [3, 1], [1, 1], [7, 1], [1, 1], [1, 1], [4, 1], [4, 0], [4, 1], [2, 1], [15, 1], [10, 0], [2, 1], [3, 1], [1, 1], [4, 1], [2, 1], [4, 1], [1, 1], [2, 1], [6, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [5, 1], [3, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [2, 1], [2, 1], [9, 1], [19, 1], [4, 1], [26, 1], [8, 1], [6, 1], [1, 1], [7, 1], [29, 1], [17, 1], [11, 1], [14, 1], [1, 1], [2, 0], [1, 1], [1, 0], [1, 1], [3, 1], [1, 1], [1, 1], [5, 1], [6, 1], [2, 1], [3, 1], [11, 1], [2, 1], [3, 1], [6, 1], [7, 1], [7, 1], [2, 1], [4, 1], [4, 1], [6, 1], [10, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [11, 1], [1, 1], [2, 1], [6, 1], [5, 1], [1, 1], [4, 1], [1, 1], [4, 1], [1, 1], [10, 1], [1, 0], [2, 1], [6, 1], [1, 1], [2, 1], [6, 1], [4, 1], [1, 1], [13, 1], [19, 1], [2, 1], [5, 1], [4, 1], [7, 1], [2, 1], [5, 1], [3, 1], [18, 1], [6, 0], [8, 0], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [2, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [3, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [2, 1], [18, 1], [14, 1], [2, 1], [2, 1], [6, 1], [7, 1], [3, 1], [3, 1], [2, 1], [8, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [7, 1], [1, 1], [2, 1], [2, 1], [3, 1], [3, 1], [1, 1], [4, 1], [3, 1], [2, 1], [5, 1], [1, 1], [1, 1], [1, 1], [8, 1], [11, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [7, 1], [1, 1], [1, 1], [23, 1], [4, 1], [5, 1], [7, 1], [5, 1], [3, 1], [4, 0], [4, 1], [5, 0], [3, 1], [8, 0], [1, 1], [10, 1], [3, 0], [5, 1], [8, 0], [2, 1], [3, 1], [2, 1], [1, 1], [2, 1], [4, 1], [4, 0], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [2, 1], [2, 1], [1, 0], [1, 1], [1, 1], [2, 0], [2, 0], [2, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [3, 1], [4, 1], [3, 1], [2, 1], [3, 1], [2, 1], [3, 1], [9, 1], [2, 1], [3, 1], [3, 1], [2, 1], [1, 1], [10, 1], [3, 1], [10, 0], [2, 1], [2, 1], [1, 1], [3, 1], [3, 1], [1, 1], [2, 1], [2, 1], [8, 0], [1, 1], [1, 1], [6, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [3, 1], [4, 1], [4, 1], [4, 1], [1, 1], [4, 1], [6, 1], [7, 1], [11, 1], [2, 1], [5, 1], [2, 1], [3, 1], [4, 1], [3, 1], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [16, 1], [3, 1], [1, 1], [1, 1], [1, 1], [3, 1], [2, 1], [3, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [9, 1], [15, 1], [7, 1], [5, 1], [11, 1], [3, 1], [2, 1], [2, 1], [2, 1], [14, 1], [4, 1], [11, 1], [2, 1], [2, 1], [4, 1], [6, 1], [2, 1], [18, 1], [4, 1], [6, 1], [4, 1], [2, 1], [6, 1], [6, 1], [3, 1], [2, 1], [1, 1], [5, 1], [4, 1], [14, 1], [9, 1], [24, 1], [24, 1], [14, 1], [7, 1], [1, 1], [2, 1], [8, 1], [31, 1], [16, 1], [14, 1], [10, 1], [19, 1], [2, 1], [3, 1], [1, 1], [2, 1], [4, 1], [2, 1], [3, 1], [2, 1], [1, 1], [8, 1], [4, 1], [8, 1], [2, 1], [1, 1], [4, 1], [12, 1], [3, 1], [2, 1], [6, 1], [9, 0], [1, 1], [5, 1], [7, 1], [3, 1], [12, 1], [9, 1], [28, 1], [7, 1], [5, 1], [11, 0], [1, 1], [3, 1], [9, 1], [1, 1], [40, 1], [12, 1], [3, 0], [11, 1], [15, 0], [19, 0], [3, 1], [39, 0], [1, 0], [2, 0], [2, 1], [15, 0], [18, 1], [3, 1], [2, 1], [1, 1], [1, 1], [6, 1], [4, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [5, 1], [4, 1], [1, 1], [1, 1], [7, 1], [1, 1], [1, 1], [9, 1], [11, 1], [7, 1], [5, 1], [5, 1], [4, 1], [4, 1], [2, 1], [2, 1], [1, 1], [11, 1], [7, 1], [4, 1], [5, 1], [6, 1], [3, 1], [1, 1], [2, 1], [1, 1], [3, 1], [2, 1], [3, 1], [21, 1], [8, 1], [5, 1], [24, 1], [4, 1], [13, 1], [7, 1], [3, 1], [3, 1], [4, 1], [8, 1], [3, 1], [10, 1], [2, 1], [4, 1], [5, 1], [2, 1], [4, 1], [22, 0], [4, 1], [1, 1], [2, 1], [3, 1], [3, 1], [1, 1], [1, 1], [1, 1], [10, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 0], [1, 1], [1, 0], [1, 1], [2, 1], [1, 1], [2, 1], [2, 1], [5, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [4, 1], [3, 1], [6, 1], [3, 1], [5, 1], [7, 1], [4, 1], [4, 1], [16, 1], [8, 1], [3, 1], [7, 1], [7, 1], [6, 1], [12, 1], [1, 1], [4, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [8, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [35, 1], [7, 1], [1, 1], [1, 1], [1, 1], [9, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [14, 1], [4, 1], [3, 1], [5, 1], [4, 1], [4, 1], [3, 1], [2, 1], [4, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [4, 1], [4, 1], [18, 1], [10, 1], [6, 1], [4, 1], [7, 1], [2, 1], [2, 1], [8, 1], [11, 1], [9, 1], [11, 1], [9, 1], [5, 1], [2, 1], [4, 1], [1, 1], [1, 1], [2, 1], [11, 1], [3, 1], [12, 1], [8, 1], [14, 1], [6, 1], [32, 1], [15, 1], [9, 1], [19, 1], [6, 1], [8, 0], [23, 1], [33, 1], [6, 1], [2, 1], [1, 1], [3, 1], [1, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [9, 1], [8, 0], [11, 0], [4, 0], [5, 1], [8, 1], [11, 1], [5, 0], [8, 1], [6, 1], [1, 0], [3, 0], [1, 0], [4, 0], [1, 0], [2, 0], [1, 0], [5, 0], [4, 0], [1, 0], [11, 0], [1, 0], [3, 1], [10, 1], [10, 1], [7, 1], [3, 1], [4, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [4, 1], [3, 1], [7, 1], [9, 1], [1, 1], [8, 1], [3, 1], [1, 1], [5, 1], [2, 1], [8, 1], [1, 1], [3, 1], [3, 1], [3, 1], [2, 1], [1, 1], [2, 1], [1, 1], [9, 1], [5, 0], [9, 1], [6, 1], [7, 0], [7, 1], [1, 0], [10, 0], [6, 0], [3, 1], [21, 0], [1, 1], [4, 0], [1, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 1], [2, 1], [1, 1], [1, 1], [3, 1], [3, 1], [3, 1], [3, 1], [1, 1], [1, 1], [3, 1], [5, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [2, 1], [2, 1], [2, 1], [2, 1], [2, 1], [4, 1], [3, 1], [3, 1], [8, 1], [3, 1], [5, 1], [3, 1], [4, 1], [11, 1], [7, 1], [2, 1], [5, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [2, 1], [9, 1], [6, 1], [2, 1], [5, 1], [16, 1], [2, 1], [3, 1], [8, 1], [4, 1], [2, 1], [5, 1], [1, 1], [9, 1], [17, 1], [11, 1], [7, 1], [5, 1], [32, 0], [8, 0], [4, 0], [6, 0], [2, 1], [2, 0], [2, 1], [1, 0], [1, 0], [7, 0], [13, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [7, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [6, 1], [3, 1], [1, 1], [4, 1], [7, 1], [2, 1], [1, 1], [2, 1], [2, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [12, 1], [5, 1], [2, 1], [14, 1], [10, 1], [14, 1], [5, 1], [1, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 0], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 0], [7, 1], [2, 1], [16, 0], [16, 1], [2, 0], [1, 1], [2, 1], [4, 1], [1, 1], [9, 1], [1, 1], [1, 1], [9, 1], [1, 1], [10, 1], [2, 1], [10, 1], [22, 0], [15, 1], [10, 0], [6, 0], [4, 1], [4, 0], [6, 1], [7, 1], [4, 0], [4, 0], [17, 0], [1, 1], [1, 1], [1, 1], [1, 1], [9, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [2, 1], [7, 1], [1, 1], [3, 1], [28, 1], [2, 1], [2, 1], [17, 1], [1, 1], [1, 1], [4, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1]]\n"
+ ]
+ }
+ ],
+ "source": [
+ "#cargando conjunto de datos\n",
+ "archivoS19Early=open(\"data/S19/Train/early.csv\")\n",
+ "\n",
+ "archivoS19Early.readline()\n",
+ "\n",
+ "datos_x=[]\n",
+ "datos_y=[]\n",
+ "\n",
+ "\n",
+ "for linea in archivoS19Early:\n",
+ " \n",
+ " linea=linea.strip(\"\\n\").split(\",\")\n",
+ " \n",
+ " subjectID=linea[0]\n",
+ " assignmentID=linea[1]\n",
+ " problemID=linea[2]\n",
+ " attempts=int(linea[3])\n",
+ " correctEventually=linea[4]\n",
+ " label=linea[5]\n",
+ " \n",
+ " if(label==\"True\"):\n",
+ " label=1\n",
+ " else:\n",
+ " label=0\n",
+ " \n",
+ " if(correctEventually==\"True\"):\n",
+ " correctEventually=1\n",
+ " else:\n",
+ " correctEventually=0\n",
+ " \n",
+ " fila=[attempts,correctEventually]\n",
+ " datos_x.append(fila)\n",
+ " \n",
+ " datos_y.append(label)\n",
+ "\n",
+ "archivoS19Early.close()\n",
+ "print(datos_x)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "id": "dac41137",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "LogisticRegression()"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#entrenando al modelo\n",
+ "regresionLogistica=sk.LogisticRegression()\n",
+ "regresionLogistica.fit(datos_x,datos_y)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "id": "eb19bc1e",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[[1, 1], [2, 1], [3, 1], [1, 1], [2, 1], [11, 1], [7, 1], [7, 1], [3, 1], [2, 1], [1, 1], [2, 1], [1, 1], [6, 1], [1, 1], [9, 1], [4, 1], [18, 1], [22, 1], [45, 1], [3, 1], [37, 1], [7, 1], [30, 1], [5, 1], [28, 1], [11, 1], [13, 1], [19, 1], [16, 1], [2, 1], [4, 1], [1, 1], [1, 1], [2, 1], [5, 1], [8, 1], [3, 1], [2, 1], [2, 1], [9, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [11, 1], [1, 1], [2, 1], [2, 1], [31, 1], [1, 1], [6, 1], [17, 1], [1, 1], [24, 1], [22, 1], [10, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [11, 1], [5, 1], [2, 1], [7, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [10, 1], [21, 1], [2, 1], [1, 1], [5, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 0], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [12, 1], [6, 1], [4, 1], [7, 1], [6, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [6, 1], [3, 1], [1, 1], [5, 1], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [2, 1], [11, 1], [4, 1], [1, 1], [4, 1], [6, 1], [2, 0], [1, 1], [5, 1], [4, 1], [4, 1], [5, 1], [10, 0], [10, 1], [3, 1], [15, 0], [8, 0], [22, 1], [5, 1], [6, 0], [6, 0], [1, 1], [14, 0], [19, 1], [9, 0], [9, 0], [2, 1], [13, 1], [4, 1], [1, 1], [12, 1], [1, 1], [2, 1], [6, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [2, 0], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [8, 1], [1, 1], [6, 1], [31, 1], [15, 1], [1, 1], [2, 1], [6, 1], [2, 1], [5, 1], [6, 1], [4, 1], [25, 1], [6, 1], [3, 1], [10, 1], [2, 1], [7, 1], [40, 0], [2, 1], [3, 1], [4, 0], [6, 0], [4, 1], [17, 1], [2, 0], [8, 1], [27, 0], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [3, 1], [13, 1], [3, 1], [1, 1], [2, 1], [3, 1], [2, 1], [1, 1], [1, 1], [5, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [3, 1], [1, 1], [19, 0], [1, 1], [2, 1], [3, 1], [1, 0], [7, 1], [5, 1], [6, 0], [7, 1], [3, 0], [2, 1], [5, 1], [18, 1], [1, 0], [10, 1], [4, 1], [3, 1], [4, 1], [1, 1], [1, 1], [14, 1], [2, 1], [1, 1], [3, 1], [1, 1], [9, 1], [1, 1], [1, 1], [1, 1], [4, 1], [9, 1], [1, 1], [2, 1], [8, 1], [12, 1], [4, 1], [2, 1], [7, 1], [13, 1], [3, 1], [2, 1], [21, 1], [3, 1], [16, 1], [33, 1], [6, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [4, 1], [2, 1], [3, 1], [1, 1], [3, 1], [3, 1], [37, 1], [14, 1], [16, 1], [18, 1], [9, 0], [7, 1], [4, 1], [6, 1], [11, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 0], [1, 1], [5, 1], [5, 1], [1, 1], [16, 1], [12, 1], [3, 1], [1, 1], [3, 1], [5, 1], [1, 1], [5, 1], [6, 1], [9, 1], [15, 1], [1, 1], [8, 0], [3, 1], [3, 1], [14, 0], [1, 1], [1, 1], [1, 0], [6, 0], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [1, 1], [11, 1], [4, 1], [14, 0], [2, 1], [41, 1], [9, 1], [10, 1], [5, 1], [2, 1], [15, 1], [5, 1], [2, 1], [2, 1], [9, 0], [9, 1], [12, 0], [3, 1], [3, 1], [7, 1], [6, 1], [10, 0], [7, 1], [1, 1], [19, 0], [4, 1], [1, 1], [10, 0], [7, 1], [12, 1], [6, 1], [7, 1], [2, 1], [1, 1], [2, 1], [3, 1], [1, 1], [22, 1], [32, 1], [3, 1], [3, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [2, 1], [3, 1], [2, 1], [4, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [8, 1], [1, 1], [4, 1], [3, 1], [1, 1], [9, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [11, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 0], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [10, 0], [8, 1], [3, 1], [4, 1], [16, 1], [11, 1], [17, 0], [17, 0], [13, 0], [4, 1], [11, 1], [5, 1], [3, 0], [18, 0], [8, 1], [11, 0], [1, 0], [4, 0], [9, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [6, 0], [9, 0], [1, 0], [3, 0], [14, 0], [1, 1], [1, 1], [3, 1], [1, 1], [18, 1], [3, 1], [2, 1], [4, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [2, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [11, 1], [2, 1], [1, 1], [4, 1], [3, 1], [1, 1], [5, 1], [2, 1], [1, 1], [1, 1], [4, 1], [16, 1], [1, 1], [1, 1], [5, 1], [1, 1], [3, 1], [3, 1], [4, 1], [3, 1], [8, 1], [2, 1], [6, 1], [1, 1], [1, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [11, 1], [8, 1], [1, 1], [13, 1], [3, 1], [2, 1], [3, 1], [4, 1], [1, 1], [2, 1], [7, 1], [9, 1], [5, 1], [1, 1], [17, 1], [19, 1], [4, 1], [21, 1], [2, 1], [3, 1], [1, 1], [12, 1], [18, 0], [4, 1], [48, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [9, 1], [4, 1], [1, 1], [2, 1], [1, 1], [5, 1], [5, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [3, 1], [3, 1], [1, 1], [5, 1], [3, 1], [2, 1], [4, 1], [1, 1], [4, 1], [1, 1], [4, 1], [3, 0], [4, 1], [3, 1], [1, 1], [5, 1], [9, 0], [18, 0], [6, 1], [4, 1], [2, 0], [9, 0], [2, 1], [16, 0], [5, 1], [3, 0], [9, 1], [6, 1], [5, 1], [3, 1], [6, 1], [1, 1], [11, 1], [10, 1], [2, 1], [2, 1], [1, 1], [4, 1], [1, 1], [6, 1], [2, 1], [3, 1], [3, 1], [1, 1], [9, 1], [6, 1], [1, 1], [3, 1], [3, 1], [3, 1], [6, 0], [1, 1], [4, 1], [2, 1], [7, 1], [4, 1], [1, 1], [17, 1], [6, 1], [11, 1], [4, 1], [3, 1], [9, 1], [8, 1], [1, 1], [2, 1], [14, 1], [16, 1], [1, 1], [7, 1], [1, 1], [1, 1], [10, 1], [3, 1], [4, 1], [6, 1], [8, 1], [2, 1], [5, 1], [11, 1], [4, 1], [1, 1], [9, 1], [6, 1], [3, 1], [1, 1], [2, 1], [6, 0], [6, 0], [6, 0], [10, 0], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [6, 1], [4, 1], [8, 1], [6, 1], [11, 1], [7, 1], [3, 1], [34, 1], [16, 1], [17, 1], [5, 1], [4, 1], [7, 1], [6, 1], [16, 1], [5, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [8, 1], [1, 1], [1, 1], [6, 1], [2, 1], [1, 1], [1, 1], [6, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [5, 1], [1, 1], [2, 1], [2, 1], [1, 1], [3, 1], [1, 1], [3, 1], [2, 1], [3, 1], [3, 1], [5, 1], [1, 1], [17, 1], [4, 1], [4, 1], [4, 1], [5, 1], [7, 1], [2, 1], [5, 1], [3, 1], [2, 1], [6, 1], [1, 1], [6, 1], [2, 1], [7, 0], [3, 1], [1, 1], [2, 1], [4, 1], [2, 1], [2, 1], [4, 1], [3, 1], [3, 1], [5, 0], [4, 1], [1, 1], [2, 1], [7, 1], [3, 1], [8, 1], [6, 1], [3, 1], [3, 1], [4, 1], [2, 1], [1, 1], [4, 1], [1, 1], [2, 1], [6, 1], [3, 1], [4, 1], [9, 0], [14, 0], [16, 1], [8, 1], [33, 0], [5, 1], [14, 1], [8, 1], [1, 1], [1, 1], [12, 0], [2, 0], [16, 1], [2, 1], [2, 1], [1, 1], [2, 1], [1, 1], [5, 1], [2, 1], [2, 1], [5, 1], [11, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [5, 1], [2, 1], [3, 1], [2, 1], [1, 1], [1, 1], [9, 1], [3, 1], [8, 1], [4, 1], [6, 1], [7, 1], [2, 1], [1, 1], [2, 1], [2, 1], [2, 1], [9, 1], [2, 1], [3, 1], [3, 1], [6, 1], [2, 1], [6, 1], [2, 1], [1, 1], [4, 1], [1, 1], [4, 1], [8, 1], [2, 1], [5, 1], [4, 1], [55, 1], [8, 1], [1, 1], [5, 1], [7, 1], [5, 1], [11, 1], [14, 1], [4, 1], [4, 1], [2, 1], [1, 1], [17, 1], [2, 1], [8, 1], [4, 1], [14, 1], [23, 1], [5, 1], [18, 1], [5, 1], [7, 1], [1, 1], [6, 1], [1, 1], [10, 1], [1, 1], [20, 1], [24, 1], [6, 1], [53, 1], [37, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 0], [1, 1], [1, 1], [3, 0], [1, 0], [1, 1], [10, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [5, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [2, 1], [2, 1], [1, 1], [3, 1], [17, 1], [4, 1], [3, 1], [5, 1], [9, 1], [12, 1], [1, 1], [13, 1], [6, 1], [8, 1], [3, 1], [5, 1], [18, 1], [4, 1], [5, 1], [6, 1], [3, 1], [2, 1], [1, 1], [2, 1], [25, 1], [2, 1], [2, 1], [2, 1], [1, 1], [8, 1], [1, 1], [4, 1], [2, 1], [1, 1], [2, 1], [4, 1], [5, 1], [52, 1], [6, 1], [46, 1], [6, 1], [8, 1], [6, 1], [24, 0], [2, 1], [15, 1], [14, 1], [16, 1], [42, 1], [8, 1], [4, 1], [2, 1], [3, 1], [2, 1], [6, 1], [9, 1], [5, 1], [10, 1], [1, 1], [2, 1], [2, 1], [5, 1], [2, 1], [1, 1], [4, 1], [7, 1], [3, 1], [3, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [4, 1], [10, 1], [1, 1], [4, 1], [2, 1], [4, 1], [1, 1], [1, 1], [6, 1], [7, 1], [1, 1], [2, 1], [15, 1], [10, 1], [4, 1], [10, 0], [9, 0], [11, 0], [6, 1], [24, 0], [1, 0], [6, 0], [11, 1], [2, 1], [4, 1], [1, 1], [1, 1], [11, 1], [8, 1], [6, 1], [1, 1], [3, 1], [5, 1], [3, 1], [3, 1], [1, 0], [4, 1], [8, 0], [3, 0], [7, 1], [16, 0], [15, 1], [9, 0], [3, 1], [2, 1], [5, 1], [10, 1], [18, 1], [5, 1], [6, 1], [5, 1], [27, 1], [3, 1], [20, 0], [21, 1], [35, 0], [8, 1], [11, 0], [3, 1], [2, 1], [6, 1], [1, 1], [1, 1], [8, 1], [1, 1], [9, 1], [2, 1], [8, 0], [2, 1], [1, 1], [2, 1], [4, 1], [1, 1], [4, 1], [6, 1], [6, 1], [1, 1], [2, 1], [10, 0], [1, 1], [3, 1], [3, 1], [1, 1], [1, 1], [3, 1], [5, 1], [2, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [7, 1], [2, 1], [1, 1], [2, 1], [4, 1], [1, 1], [7, 1], [7, 1], [3, 1], [1, 1], [3, 1], [4, 1], [3, 1], [4, 1], [1, 1], [8, 1], [10, 1], [3, 1], [10, 1], [8, 1], [9, 1], [10, 1], [6, 1], [3, 1], [4, 1], [2, 1], [1, 1], [2, 1], [2, 1], [4, 1], [6, 1], [6, 1], [4, 1], [7, 1], [1, 1], [3, 1], [2, 1], [3, 1], [1, 1], [1, 1], [3, 1], [3, 1], [2, 1], [1, 1], [1, 1], [3, 1], [3, 1], [1, 1], [16, 1], [3, 1], [5, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [3, 1], [2, 1], [9, 1], [2, 1], [4, 1], [11, 1], [21, 1], [7, 1], [14, 1], [7, 1], [7, 1], [3, 1], [2, 1], [26, 1], [18, 1], [20, 1], [11, 1], [1, 1], [1, 1], [2, 1], [1, 1], [19, 1], [2, 1], [1, 1], [8, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [5, 1], [4, 1], [1, 1], [12, 1], [11, 1], [2, 1], [6, 1], [12, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [12, 1], [5, 1], [1, 1], [3, 1], [18, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [6, 1], [1, 1], [8, 1], [1, 1], [3, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [2, 1], [2, 1], [5, 1], [1, 1], [1, 1], [7, 1], [4, 1], [8, 1], [4, 1], [2, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [6, 1], [2, 1], [7, 1], [1, 1], [7, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [12, 1], [7, 1], [5, 1], [8, 0], [11, 1], [4, 1], [3, 1], [1, 0], [1, 1], [9, 1], [1, 1], [2, 1], [6, 1], [9, 0], [12, 0], [2, 1], [3, 1], [6, 1], [4, 1], [4, 1], [4, 1], [2, 1], [1, 1], [12, 1], [3, 1], [14, 1], [10, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [12, 1], [1, 1], [2, 1], [14, 1], [2, 1], [3, 1], [3, 1], [3, 1], [4, 1], [1, 1], [2, 1], [3, 1], [1, 1], [11, 1], [4, 1], [1, 1], [2, 1], [9, 1], [9, 1], [8, 1], [5, 1], [10, 1], [9, 0], [2, 1], [1, 1], [8, 1], [3, 1], [2, 1], [5, 1], [2, 1], [8, 1], [4, 1], [2, 1], [4, 1], [11, 1], [7, 1], [1, 1], [8, 1], [6, 1], [17, 1], [6, 1], [4, 0], [5, 1], [8, 1], [9, 0], [5, 1], [8, 1], [37, 0], [13, 0], [4, 0], [10, 0], [5, 0], [11, 0], [3, 1], [3, 1], [1, 1], [6, 1], [2, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [4, 1], [16, 1], [12, 1], [7, 0], [5, 1], [6, 0], [12, 0], [2, 1], [14, 1], [10, 1], [7, 1], [2, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [7, 1], [12, 1], [6, 1], [8, 1], [21, 0], [7, 1], [23, 1], [10, 1], [2, 1], [3, 1], [6, 1], [2, 1], [12, 1], [13, 1], [15, 1], [4, 1], [1, 1], [21, 1], [6, 1], [8, 1], [39, 1], [1, 1], [4, 1], [9, 1], [1, 1], [1, 1], [1, 1], [4, 1], [4, 1], [1, 1], [3, 1], [1, 1], [2, 1], [3, 1], [6, 0], [1, 1], [3, 0], [2, 1], [1, 1], [9, 1], [3, 1], [1, 1], [4, 1], [3, 0], [1, 1], [2, 1], [3, 1], [3, 1], [1, 1], [2, 1], [5, 1], [1, 1], [1, 1], [8, 1], [6, 1], [1, 1], [1, 1], [1, 1], [10, 1], [1, 1], [7, 1], [4, 1], [9, 1], [6, 1], [5, 1], [2, 1], [2, 1], [9, 1], [8, 1], [11, 1], [1, 1], [2, 1], [12, 1], [1, 1], [3, 1], [2, 1], [3, 1], [5, 1], [4, 1], [4, 1], [1, 1], [2, 1], [3, 1], [15, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 1], [8, 1], [2, 1], [8, 1], [7, 1], [3, 1], [3, 1], [22, 1], [1, 1], [4, 1], [7, 1], [4, 1], [3, 1], [4, 1], [1, 1], [1, 1], [16, 1], [1, 1], [3, 1], [7, 1], [1, 1], [6, 1], [16, 1], [3, 1], [2, 1], [6, 1], [2, 1], [4, 1], [11, 1], [8, 1], [2, 1], [3, 1], [31, 1], [5, 1], [12, 1], [2, 1], [5, 1], [7, 1], [1, 1], [12, 1], [3, 1], [1, 1], [1, 1], [7, 1], [13, 1], [4, 1], [6, 1], [21, 1], [3, 1], [1, 1], [6, 1], [2, 1], [3, 1], [2, 1], [3, 1], [2, 1], [4, 1], [3, 1], [2, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [4, 1], [1, 1], [7, 1], [9, 1], [22, 1], [2, 1], [12, 0], [9, 1], [3, 1], [20, 1], [14, 1], [22, 1], [21, 1], [2, 1], [8, 1], [7, 1], [2, 1], [1, 1], [1, 1], [1, 1], [6, 1], [3, 1], [4, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [19, 1], [2, 1], [1, 1], [6, 1], [23, 1], [2, 1], [8, 1], [5, 1], [1, 1], [8, 1], [16, 1], [20, 1], [13, 1], [15, 1], [12, 1], [8, 1], [1, 1], [2, 1], [2, 1], [5, 1], [2, 1], [7, 1], [3, 1], [3, 1], [1, 1], [9, 1], [9, 0], [6, 0], [4, 1], [1, 1], [7, 1], [11, 1], [1, 1], [4, 1], [6, 1], [6, 1], [2, 1], [5, 1], [3, 1], [6, 0], [9, 0], [1, 0], [6, 1], [7, 0], [11, 1], [3, 1], [6, 0], [20, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [3, 1], [3, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [5, 1], [13, 1], [8, 1], [26, 1], [1, 1], [3, 1], [7, 1], [1, 1], [6, 1], [7, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [3, 1], [4, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [2, 1], [1, 1], [11, 1], [7, 1], [3, 1], [8, 1], [13, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [3, 1], [1, 1], [4, 1], [3, 1], [11, 1], [18, 1], [41, 1], [4, 1], [4, 1], [6, 1], [2, 1], [1, 1], [8, 1], [4, 1], [1, 1], [19, 1], [20, 1], [21, 1], [11, 1], [10, 1], [35, 0], [71, 1], [3, 1], [10, 0], [11, 0], [32, 0], [8, 1], [4, 1], [55, 1], [2, 1], [3, 1], [8, 1], [6, 1], [4, 1], [2, 1], [1, 1], [13, 1], [5, 1], [1, 1], [3, 1], [1, 1], [1, 1], [5, 1], [3, 1], [2, 1], [8, 1], [3, 1], [3, 1], [6, 1], [7, 1], [10, 1], [19, 1], [5, 1], [7, 0], [5, 1], [9, 1], [4, 1], [1, 1], [15, 0], [5, 1], [21, 1], [2, 1], [2, 1], [2, 1], [3, 1], [1, 1], [11, 1], [9, 1], [12, 1], [3, 1], [4, 1], [2, 1], [2, 1], [3, 1], [17, 1], [9, 1], [9, 1], [49, 1], [5, 1], [10, 1], [93, 1], [19, 1], [10, 1], [14, 1], [28, 1], [51, 1], [39, 1], [11, 1], [3, 1], [13, 1], [19, 1], [21, 1], [3, 1], [2, 1], [2, 1], [1, 1], [23, 1], [1, 1], [4, 1], [2, 1], [7, 1], [3, 1], [3, 1], [10, 1], [3, 1], [2, 1], [3, 1], [2, 1], [6, 1], [2, 1], [11, 1], [6, 1], [13, 1], [4, 1], [2, 1], [2, 1], [5, 1], [6, 1], [13, 1], [2, 1], [9, 1], [21, 1], [14, 0], [3, 1], [1, 0], [6, 1], [2, 1], [2, 1], [2, 1], [4, 0], [3, 0], [4, 0], [2, 1], [2, 1], [11, 1], [1, 1], [10, 1], [2, 1], [1, 1], [2, 1], [4, 1], [9, 1], [1, 1], [4, 1], [1, 1], [4, 1], [10, 1], [7, 1], [1, 1], [5, 1], [10, 1], [25, 1], [7, 1], [5, 1], [8, 1], [4, 1], [1, 1], [1, 1], [4, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [9, 0], [2, 1], [3, 0], [4, 1], [1, 1], [1, 1], [11, 1], [1, 1], [1, 1], [4, 1], [1, 1], [3, 1], [6, 1], [2, 1], [13, 1], [6, 1], [3, 0], [10, 0], [8, 1], [28, 1], [10, 1], [13, 0], [3, 0], [3, 0], [19, 0], [10, 0], [9, 1], [17, 1], [3, 1], [2, 1], [2, 1], [1, 1], [8, 1], [4, 1], [1, 1], [2, 1], [5, 1], [1, 1], [3, 1], [6, 1], [1, 1], [1, 1], [4, 1], [2, 1], [2, 1], [7, 1], [9, 0], [18, 0], [4, 1], [27, 0], [6, 1], [9, 0], [1, 1], [2, 1], [11, 1], [17, 1], [12, 0], [7, 1], [3, 1], [6, 1], [7, 1], [2, 1], [2, 1], [1, 1], [1, 1], [3, 1], [1, 1], [5, 1], [3, 1], [4, 1], [6, 1], [1, 1], [6, 1], [1, 1], [5, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [5, 1], [3, 1], [1, 1], [5, 1], [1, 1], [1, 1], [5, 1], [1, 1], [3, 1], [1, 1], [3, 1], [1, 1], [2, 1], [2, 1], [9, 0], [1, 1], [7, 1], [1, 1], [2, 1], [3, 1], [1, 1], [3, 1], [1, 1], [19, 1], [2, 1], [3, 1], [4, 1], [17, 1], [7, 1], [1, 1], [5, 1], [1, 1], [6, 1], [15, 1], [6, 1], [7, 1], [7, 1], [8, 1], [6, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [7, 1], [4, 1], [1, 1], [6, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [22, 1], [6, 1], [1, 1], [5, 1], [1, 1], [9, 1], [1, 1], [1, 1], [2, 1], [3, 1], [2, 1], [13, 1], [4, 1], [3, 1], [3, 1], [12, 1], [1, 1], [14, 1], [6, 1], [12, 1], [2, 1], [7, 1], [12, 1], [5, 1], [9, 1], [10, 1], [2, 1], [6, 1], [10, 1], [1, 1], [18, 1], [6, 1], [2, 1], [3, 1], [3, 1], [1, 1], [4, 1], [2, 1], [1, 1], [2, 1], [7, 1], [14, 1], [2, 1], [14, 1], [19, 1], [28, 1], [2, 1], [6, 1], [21, 1], [5, 1], [7, 1], [8, 1], [2, 1], [43, 1], [35, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [3, 1], [2, 1], [3, 1], [4, 1], [6, 1], [3, 1], [19, 0], [2, 1], [4, 1], [2, 1], [2, 1], [7, 1], [3, 1], [5, 1], [2, 1], [2, 1], [2, 1], [1, 1], [5, 1], [5, 1], [2, 1], [3, 1], [1, 1], [6, 1], [19, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [13, 1], [5, 0], [1, 1], [2, 1], [1, 1], [2, 1], [3, 1], [3, 1], [6, 1], [3, 1], [5, 1], [7, 0], [4, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [4, 1], [1, 1], [4, 1], [11, 1], [1, 1], [1, 1], [2, 1], [4, 1], [7, 1], [5, 1], [3, 1], [12, 1], [2, 1], [2, 1], [11, 1], [5, 1], [36, 1], [16, 1], [10, 1], [24, 0], [12, 1], [42, 1], [3, 1], [1, 1], [7, 1], [13, 0], [9, 0], [10, 1], [3, 1], [5, 1], [1, 1], [10, 1], [7, 1], [8, 1], [1, 1], [2, 1], [3, 1], [2, 1], [1, 1], [1, 1], [5, 1], [1, 1], [8, 1], [3, 1], [1, 1], [2, 1], [1, 0], [2, 1], [3, 1], [1, 1], [2, 1], [1, 1], [6, 0], [3, 1], [8, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 1], [5, 1], [1, 1], [10, 1], [2, 1], [5, 1], [4, 1], [6, 1], [12, 1], [16, 0], [5, 1], [11, 0], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [4, 1], [1, 1], [1, 1], [4, 1], [37, 1], [1, 1], [1, 1], [18, 1], [3, 1], [4, 1], [1, 1], [1, 1], [2, 1], [9, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [40, 1], [5, 1], [1, 1], [1, 1], [2, 1], [1, 1], [5, 1], [14, 1], [15, 1], [14, 1], [3, 1], [4, 1], [1, 0], [4, 1], [5, 1], [11, 1], [2, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [5, 1], [2, 1], [3, 1], [9, 0], [15, 0], [3, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [9, 1], [3, 1], [10, 1], [4, 1], [2, 1], [1, 1], [4, 1], [1, 1], [16, 1], [1, 1], [1, 1], [4, 1], [4, 1], [4, 1], [1, 1], [3, 1], [1, 1], [1, 1], [8, 1], [3, 1], [1, 1], [1, 1], [4, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [5, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 0], [1, 1], [3, 0], [7, 1], [8, 0], [10, 0], [1, 1], [5, 0], [2, 0], [2, 1], [5, 1], [1, 1], [4, 1], [2, 1], [1, 1], [9, 1], [7, 1], [8, 1], [3, 1], [4, 1], [7, 1], [1, 1], [4, 1], [6, 1], [3, 1], [1, 1], [6, 1], [11, 1], [3, 1], [1, 1], [2, 1], [2, 1], [15, 1], [2, 1], [4, 1], [1, 1], [1, 1], [5, 1], [11, 1], [2, 1], [5, 1], [18, 1], [3, 1], [3, 1], [2, 1], [1, 1], [10, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [6, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 0], [3, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [10, 0], [3, 1], [2, 1], [7, 0], [3, 1], [1, 1], [1, 1], [3, 1], [3, 1], [2, 1], [10, 1], [6, 0], [2, 0], [3, 1], [9, 0], [6, 0], [4, 1], [1, 1], [4, 1], [3, 1], [1, 1], [3, 1], [12, 0], [1, 1], [9, 0], [7, 0], [6, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [6, 1], [1, 1], [3, 1], [3, 1], [9, 1], [8, 0], [12, 0], [8, 0], [3, 1], [13, 0], [9, 1], [7, 0], [3, 0], [10, 1], [6, 0], [5, 0], [1, 0], [1, 0], [1, 0], [2, 0], [1, 0], [3, 0], [1, 0], [1, 1], [2, 1], [2, 1], [1, 1], [7, 1], [5, 1], [1, 1], [1, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [2, 1], [6, 1], [1, 1], [1, 1], [4, 1], [4, 1], [1, 1], [1, 1], [3, 1], [3, 1], [2, 1], [3, 1], [1, 1], [2, 1], [1, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [5, 1], [2, 1], [4, 1], [14, 1], [15, 1], [1, 1], [2, 1], [7, 1], [3, 1], [1, 1], [3, 1], [13, 1], [7, 1], [8, 1], [3, 1], [1, 1], [1, 1], [4, 1], [2, 1], [2, 1], [4, 1], [7, 1], [3, 1], [2, 1], [1, 1], [3, 1], [4, 1], [5, 1], [1, 1], [3, 1], [1, 1], [1, 1], [2, 1], [11, 1], [3, 1], [2, 1], [10, 1], [5, 1], [6, 1], [3, 1], [1, 1], [14, 1], [3, 1], [13, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [15, 1], [8, 1], [4, 0], [2, 1], [2, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [6, 1], [6, 1], [12, 1], [5, 1], [5, 0], [10, 1], [1, 1], [13, 0], [6, 0], [4, 0], [5, 0], [2, 1], [6, 0], [9, 1], [7, 1], [10, 1], [1, 1], [12, 1], [8, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [2, 1], [2, 1], [3, 1], [3, 1], [3, 1], [2, 1], [9, 1], [3, 1], [1, 1], [9, 1], [14, 1], [4, 1], [12, 1], [4, 1], [19, 1], [5, 1], [2, 1], [12, 1], [3, 1], [7, 1], [5, 1], [1, 1], [6, 1], [11, 1], [2, 1], [4, 1], [2, 1], [1, 1], [3, 1], [1, 1], [6, 1], [1, 1], [4, 1], [2, 1], [9, 1], [13, 1], [3, 1], [7, 1], [1, 1], [7, 1], [26, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [2, 1], [1, 1], [3, 1], [2, 1], [2, 1], [5, 1], [4, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [4, 1], [2, 1], [4, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [2, 1], [3, 0], [2, 1], [2, 1], [9, 0], [3, 0], [2, 0], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [6, 0], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [4, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 1], [4, 1], [7, 1], [3, 1], [1, 1], [5, 1], [10, 1], [6, 1], [16, 1], [2, 1], [3, 1], [5, 1], [7, 1], [8, 1], [4, 1], [7, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [19, 1], [9, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [3, 1], [2, 1], [8, 1], [21, 1], [9, 1], [3, 1], [8, 1], [9, 1], [1, 1], [5, 1], [2, 1], [4, 1], [9, 1], [7, 1], [8, 1], [6, 1], [11, 1], [33, 1], [9, 1], [21, 1], [9, 1], [2, 1], [6, 1], [7, 1], [60, 1], [10, 1], [38, 1], [5, 1], [2, 1], [1, 1], [1, 1], [2, 1], [7, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [9, 1], [2, 1], [1, 1], [1, 1], [7, 0], [1, 1], [5, 0], [4, 1], [1, 1], [7, 1], [1, 1], [4, 1], [9, 0], [13, 1], [14, 1], [6, 0], [12, 0], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [2, 1], [5, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [8, 1], [18, 0], [5, 1], [6, 1], [5, 1], [27, 0], [3, 1], [14, 0], [11, 1], [2, 1], [3, 1], [26, 1], [5, 1], [6, 1], [15, 1], [1, 1], [5, 1], [1, 1], [1, 1], [1, 1], [10, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [11, 1], [1, 1], [2, 1], [25, 1], [17, 1], [2, 1], [1, 1], [8, 1], [8, 1], [9, 1], [14, 1], [10, 1], [3, 1], [1, 1], [8, 1], [14, 1], [34, 1], [21, 0], [13, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [4, 1], [5, 0], [8, 1], [1, 1], [3, 1], [35, 0], [3, 1], [3, 1], [7, 0], [3, 1], [7, 0], [2, 1], [1, 1], [2, 0], [2, 1], [2, 1], [4, 0], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [9, 1], [1, 1], [1, 1], [8, 1], [18, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [4, 1], [2, 1], [1, 1], [7, 1], [9, 1], [3, 1], [1, 1], [4, 1], [3, 1], [1, 1], [4, 1], [1, 1], [2, 1], [5, 1], [6, 1], [6, 1], [1, 1], [8, 1], [5, 1], [3, 1], [5, 1], [15, 1], [5, 1], [2, 1], [7, 1], [4, 1], [3, 1], [3, 1], [7, 1], [1, 1], [1, 1], [1, 1], [1, 1], [6, 1], [5, 1], [1, 0], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [2, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [6, 1], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [2, 1], [14, 1], [7, 1], [5, 1], [5, 1], [24, 1], [1, 1], [5, 1], [7, 1], [1, 1], [1, 1], [5, 1], [3, 1], [6, 1], [8, 1], [1, 1], [1, 1], [2, 1], [6, 1], [5, 1], [4, 1], [2, 1], [2, 1], [4, 1], [3, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [8, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [2, 1], [1, 1], [2, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [3, 0], [3, 1], [10, 1], [36, 1], [3, 1], [8, 0], [3, 1], [9, 0], [8, 0], [1, 0], [5, 0], [6, 1], [14, 0], [7, 1], [1, 1], [8, 0], [18, 0], [6, 1], [12, 1], [14, 0], [21, 0], [5, 1], [2, 1], [1, 1], [2, 1], [1, 1], [4, 1], [6, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [4, 1], [1, 1], [5, 1], [4, 1], [1, 1], [2, 1], [4, 1], [13, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [6, 1], [4, 1], [4, 1], [2, 1], [8, 1], [3, 1], [6, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [5, 0], [6, 0], [2, 1], [4, 1], [15, 1], [8, 1], [10, 1], [5, 1], [1, 1], [3, 1], [3, 1], [4, 1], [5, 0], [6, 1], [7, 1], [1, 1], [9, 1], [1, 1], [1, 1], [1, 1], [7, 1], [4, 1], [2, 1], [6, 1], [1, 1], [1, 1], [1, 1], [2, 1], [5, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [4, 1], [3, 0], [2, 1], [2, 1], [20, 1], [29, 1], [1, 1], [8, 1], [6, 1], [17, 1], [1, 1], [2, 1], [1, 1], [6, 1], [20, 1], [2, 1], [8, 1], [5, 1], [4, 0], [17, 0], [15, 1], [3, 0], [7, 0], [6, 0], [1, 1], [31, 0], [11, 0], [7, 0], [3, 0], [6, 1], [6, 1], [2, 1], [3, 1], [10, 1], [9, 1], [3, 1], [3, 1], [9, 1], [3, 1], [2, 1], [4, 1], [1, 1], [7, 1], [26, 1], [3, 1], [5, 1], [3, 1], [4, 0], [3, 1], [3, 1], [3, 1], [4, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [2, 1], [2, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [3, 1], [1, 1], [1, 1], [3, 1], [32, 1], [1, 1], [2, 1], [21, 1], [51, 1], [9, 1], [7, 1], [8, 1], [18, 1], [13, 1], [10, 1], [1, 1], [5, 1], [6, 1], [9, 0], [3, 1], [2, 1], [3, 0], [16, 0], [3, 1], [8, 1], [6, 1], [4, 0], [7, 0], [14, 0], [8, 1], [9, 1], [12, 1], [6, 0], [22, 0], [18, 1], [6, 1], [3, 1], [10, 1], [10, 1], [14, 1], [35, 1], [12, 1], [4, 1], [58, 1], [2, 1], [4, 1], [11, 1], [30, 1], [11, 1], [2, 1], [13, 1], [5, 1], [30, 1], [5, 1], [1, 1], [6, 1], [7, 1], [3, 1], [11, 1], [12, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [10, 1], [1, 1], [1, 1], [3, 1], [5, 1], [12, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [8, 1], [1, 1], [1, 1], [12, 1], [5, 1], [1, 1], [44, 1], [3, 1], [8, 1], [1, 1], [5, 1], [4, 1], [9, 1], [5, 1], [2, 1], [5, 1], [4, 1], [35, 1], [2, 1], [19, 1], [67, 0], [26, 1], [11, 1], [66, 0], [14, 1], [24, 1], [11, 1], [2, 1], [11, 1], [15, 1], [14, 1], [10, 1], [2, 1], [5, 1], [6, 1], [2, 1], [4, 1], [20, 1], [1, 1], [5, 1], [25, 1], [10, 1], [1, 1], [8, 1], [4, 1], [12, 1], [5, 1], [3, 1], [23, 1], [2, 1], [7, 1], [11, 0], [4, 1], [5, 1], [5, 1], [1, 1], [19, 0], [1, 1], [10, 1], [12, 1], [3, 1], [2, 1], [2, 1], [1, 1], [8, 1], [1, 1], [9, 1], [5, 1], [5, 1], [2, 1], [9, 1], [3, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [2, 1], [2, 1], [5, 1], [1, 1], [4, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [15, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [6, 1], [3, 1], [4, 1], [2, 1], [1, 1], [1, 0], [1, 1], [10, 1], [2, 1], [10, 0], [4, 1], [5, 1], [7, 0], [5, 1], [4, 1], [8, 1], [3, 1], [3, 1], [2, 1], [1, 1], [4, 1], [3, 1], [1, 1], [1, 1], [3, 1], [2, 1], [7, 1], [5, 1], [8, 1], [1, 1], [39, 1], [16, 1], [2, 1], [3, 1], [16, 1], [13, 1], [1, 1], [6, 1], [2, 1], [2, 1], [5, 1], [3, 1], [3, 1], [11, 1], [9, 1], [1, 1], [1, 1], [3, 1], [4, 1], [2, 1], [12, 1], [4, 1], [4, 1], [2, 1], [6, 1], [2, 1], [1, 1], [1, 1], [3, 1], [2, 1], [2, 1], [11, 1], [2, 1], [21, 1], [16, 1], [6, 1], [2, 1], [2, 1], [1, 1], [8, 1], [1, 1], [1, 1], [2, 0], [12, 0], [19, 0], [1, 1], [11, 0], [8, 0], [3, 1], [3, 1], [3, 1], [15, 1], [5, 1], [5, 1], [4, 1], [2, 1], [6, 1], [13, 0], [2, 0], [2, 1], [8, 1], [4, 0], [6, 0], [3, 1], [2, 1], [1, 1], [1, 1], [5, 1], [2, 1], [5, 1], [1, 1], [3, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [4, 1], [2, 1], [4, 1], [11, 1], [19, 1], [2, 1], [27, 1], [4, 1], [3, 1], [13, 1], [13, 1], [15, 1], [7, 1], [8, 0], [9, 1], [1, 1], [3, 1], [1, 1], [6, 1], [4, 1], [4, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [5, 1], [1, 1], [1, 1], [5, 1], [4, 1], [3, 1], [2, 1], [3, 1], [1, 1], [1, 1], [2, 1], [3, 1], [8, 1], [5, 1], [2, 1], [2, 1], [1, 1], [4, 0], [8, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [7, 0], [5, 1], [6, 1], [4, 1], [1, 1], [2, 1], [9, 1], [3, 1], [12, 1], [5, 1], [4, 1], [3, 1], [1, 1], [11, 1], [1, 1], [1, 1], [1, 1], [1, 1], [10, 1], [3, 1], [2, 1], [2, 1], [9, 1], [26, 1], [2, 1], [11, 1], [1, 1], [7, 1], [4, 1], [10, 1], [25, 1], [40, 1], [25, 1], [6, 1], [2, 1], [3, 1], [2, 1], [1, 1], [5, 1], [16, 1], [1, 1], [12, 1], [4, 1], [6, 1], [12, 1], [10, 1], [6, 1], [15, 0], [2, 1], [8, 1], [21, 1], [3, 1], [10, 1], [21, 1], [3, 1], [8, 1], [6, 1], [1, 1], [8, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [2, 1], [3, 1], [7, 1], [2, 1], [1, 1], [2, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [3, 1], [1, 1], [6, 1], [8, 1], [17, 1], [2, 1], [3, 1], [1, 1], [14, 1], [8, 1], [3, 1], [9, 1], [32, 1], [7, 1], [7, 1], [3, 1], [2, 1], [43, 1], [5, 1], [2, 1], [26, 1], [3, 1], [3, 1], [3, 1], [30, 1], [14, 1], [3, 1], [5, 1], [7, 1], [6, 1], [12, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [2, 1], [1, 1], [5, 1], [1, 1], [1, 1], [1, 1], [8, 1], [2, 1], [5, 1], [10, 1], [5, 1], [3, 1], [2, 1], [1, 0], [9, 1], [3, 1], [18, 1], [4, 1], [3, 1], [1, 1], [3, 1], [9, 1], [1, 1], [2, 1], [20, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [3, 1], [1, 1], [3, 1], [7, 1], [1, 1], [2, 1], [2, 1], [4, 1], [1, 1], [6, 1], [6, 1], [8, 1], [6, 1], [2, 1], [7, 1], [2, 1], [3, 1], [5, 1], [3, 1], [5, 1], [1, 1], [6, 1], [7, 1], [3, 1], [20, 1], [12, 1], [16, 1], [11, 1], [9, 1], [4, 1], [8, 1], [26, 1], [29, 1], [12, 1], [4, 1], [7, 1], [9, 1], [6, 1], [2, 1], [2, 0], [4, 1], [1, 1], [29, 0], [44, 0], [2, 1], [7, 1], [3, 1], [3, 1], [1, 1], [4, 1], [1, 1], [8, 1], [9, 0], [13, 1], [21, 1], [8, 1], [26, 0], [17, 1], [16, 1], [10, 1], [3, 1], [1, 0], [9, 1], [6, 1], [4, 1], [15, 1], [11, 1], [13, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [7, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [5, 0], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [6, 1], [1, 1], [2, 1], [4, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [6, 1], [3, 1], [1, 1], [1, 1], [9, 1], [1, 1], [5, 1], [2, 1], [13, 0], [1, 1], [5, 1], [9, 1], [2, 1], [6, 1], [1, 1], [2, 1], [2, 1], [2, 1], [2, 1], [4, 1], [14, 1], [3, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [10, 1], [1, 1], [1, 1], [2, 1], [1, 0], [1, 1], [1, 1], [7, 1], [1, 1], [2, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [28, 1], [6, 1], [7, 1], [2, 1], [5, 1], [2, 1], [10, 1], [6, 1], [7, 1], [1, 1], [11, 1], [5, 1], [1, 1], [6, 1], [10, 1], [12, 1], [1, 1], [2, 1], [4, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [2, 1], [1, 1], [6, 1], [1, 1], [2, 1], [4, 1], [3, 1], [9, 1], [1, 1], [6, 1], [1, 1], [17, 1], [4, 1], [1, 1], [1, 1], [2, 1], [5, 1], [9, 1], [4, 1], [1, 1], [3, 1], [1, 1], [6, 1], [4, 1], [7, 1], [1, 1], [1, 1], [4, 1], [2, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [2, 1], [3, 1], [5, 1], [3, 1], [2, 1], [6, 1], [2, 1], [4, 1], [5, 1], [3, 1], [2, 1], [1, 1], [6, 1], [5, 1], [8, 1], [6, 0], [3, 1], [1, 1], [4, 1], [2, 1], [2, 1], [3, 0], [2, 1], [9, 1], [2, 1], [2, 1], [4, 1], [1, 1], [35, 1], [5, 1], [3, 1], [2, 1], [2, 1], [3, 1], [2, 1], [2, 1], [1, 1], [4, 1], [16, 1], [8, 1], [14, 1], [3, 1], [6, 1], [12, 1], [7, 1], [61, 0], [31, 1], [67, 0], [49, 1], [6, 1], [7, 1], [4, 1], [24, 1], [16, 1], [3, 1], [3, 0], [3, 1], [1, 1], [8, 0], [27, 0], [1, 1], [3, 1], [3, 1], [7, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [2, 1], [3, 1], [9, 0], [1, 1], [15, 1], [1, 1], [3, 1], [12, 0], [4, 0], [3, 1], [3, 1], [7, 0], [10, 1], [1, 1], [3, 1], [1, 1], [3, 1], [7, 1], [9, 1], [5, 1], [2, 1], [14, 1], [17, 0], [3, 1], [15, 1], [18, 1], [11, 0], [24, 1], [10, 1], [13, 1], [6, 1], [8, 1], [14, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [10, 1], [1, 1], [1, 1], [2, 1], [1, 1], [24, 1], [3, 1], [12, 1], [4, 1], [6, 1], [24, 1], [22, 1], [9, 1], [14, 0], [27, 1], [4, 0], [6, 1], [3, 1], [8, 1], [27, 1], [3, 1], [3, 1], [1, 1], [3, 1], [2, 1], [3, 1], [10, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [6, 1], [1, 1], [1, 1], [8, 1], [6, 1], [6, 1], [5, 1], [2, 1], [9, 1], [3, 1], [1, 1], [4, 1], [4, 1], [3, 1], [4, 1], [10, 1], [1, 1], [10, 1], [2, 1], [1, 1], [5, 1], [7, 1], [5, 1], [5, 1], [2, 1], [6, 1], [7, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [1, 1], [6, 1], [1, 1], [4, 1], [1, 1], [1, 1], [6, 1], [5, 1], [3, 1], [9, 1], [1, 1], [1, 1], [2, 1], [5, 1], [3, 1], [4, 1], [7, 1], [7, 1], [3, 1], [3, 1], [6, 1], [8, 1], [1, 0], [8, 0], [3, 1], [8, 1], [2, 1], [7, 1], [12, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 0], [1, 1], [1, 1], [5, 1], [10, 1], [10, 1], [27, 1], [4, 1], [2, 1], [2, 1], [2, 1], [4, 1], [1, 1], [3, 1], [2, 1], [4, 1], [3, 1], [1, 1], [4, 1], [6, 1], [10, 1], [23, 1], [1, 1], [12, 1], [3, 1], [23, 1], [5, 1], [6, 1], [14, 1], [16, 1], [15, 1], [13, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [3, 1], [5, 1], [3, 1], [2, 1], [1, 1], [7, 1], [6, 1], [1, 1], [7, 1], [48, 1], [1, 1], [9, 1], [3, 1], [3, 1], [3, 1], [2, 1], [10, 1], [3, 1], [6, 1], [1, 1], [6, 1], [2, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [5, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [14, 1], [1, 1], [3, 1], [3, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [5, 1], [2, 1], [1, 1], [3, 1], [1, 1], [6, 1], [4, 1], [1, 1], [11, 1], [12, 1], [6, 1], [3, 1], [2, 1], [3, 0], [1, 1], [7, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [5, 1], [2, 1], [3, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [13, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [12, 0], [14, 0], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [3, 0], [3, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [2, 1], [3, 1], [2, 1], [3, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [3, 1], [1, 1], [3, 1], [3, 1], [3, 1], [1, 1], [13, 1], [1, 1], [7, 1], [10, 1], [6, 1], [2, 1], [2, 1], [2, 1], [1, 1], [2, 1], [2, 1], [2, 1], [2, 1], [1, 1], [4, 1], [2, 1], [6, 1], [2, 1], [4, 1], [4, 1], [15, 1], [6, 1], [3, 1], [5, 0], [13, 1], [5, 1], [22, 0], [1, 1], [7, 1], [2, 1], [3, 1], [3, 1], [9, 1], [5, 1], [4, 0], [2, 1], [1, 1], [1, 1], [1, 1], [4, 1], [3, 1], [3, 1], [2, 1], [3, 1], [4, 1], [1, 1], [3, 1], [1, 1], [1, 1], [4, 1], [6, 1], [2, 1], [9, 1], [4, 1], [22, 1], [2, 1], [4, 0], [1, 1], [3, 0], [2, 1], [1, 1], [2, 1], [6, 1], [4, 1], [15, 0], [3, 1], [2, 1], [2, 1], [3, 1], [8, 1], [4, 1], [3, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [6, 0], [1, 1], [1, 1], [7, 1], [1, 1], [2, 1], [2, 1], [4, 1], [1, 1], [1, 1], [1, 1], [8, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [12, 1], [1, 1], [10, 1], [3, 1], [9, 1], [13, 0], [6, 1], [8, 1], [5, 1], [1, 1], [7, 1], [4, 1], [1, 1], [3, 1], [4, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [4, 1], [1, 1], [9, 1], [2, 1], [2, 1], [7, 1], [58, 0], [2, 1], [11, 1], [1, 1], [4, 1], [1, 1], [4, 1], [10, 1], [4, 1], [28, 1], [3, 1], [1, 1], [1, 1], [2, 1], [1, 1], [3, 1], [5, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 0], [1, 1], [1, 1], [1, 1], [1, 1], [6, 0], [5, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [4, 1], [4, 1], [5, 1], [17, 1], [1, 1], [3, 1], [9, 1], [5, 1], [1, 1], [7, 1], [4, 1], [4, 1], [5, 1], [16, 1], [30, 1], [9, 1], [7, 1], [40, 1], [9, 1], [6, 1], [4, 1], [34, 1], [16, 1], [6, 1], [16, 1], [14, 1], [21, 1], [26, 1], [7, 1], [19, 0], [18, 1], [9, 1], [23, 0], [24, 0], [23, 1], [1, 1], [4, 1], [6, 1], [1, 1], [3, 1], [1, 1], [11, 0], [10, 0], [3, 0], [2, 0], [12, 0], [1, 0], [7, 0], [1, 0], [1, 0], [2, 0], [3, 0], [1, 0], [7, 0], [1, 0], [1, 0], [2, 1], [1, 1], [1, 1], [1, 1], [9, 1], [3, 1], [3, 1], [1, 1], [2, 1], [2, 1], [1, 1], [3, 1], [1, 1], [1, 1], [2, 1], [1, 0], [5, 0], [7, 1], [2, 0], [4, 0], [5, 0], [2, 1], [3, 0], [4, 1], [7, 1], [3, 0], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [5, 1], [2, 1], [1, 1], [1, 1], [3, 1], [1, 1], [2, 1], [3, 1], [2, 1], [2, 1], [2, 1], [2, 1], [1, 1], [8, 1], [19, 0], [2, 1], [34, 0], [4, 1], [12, 0], [3, 1], [2, 1], [11, 1], [8, 0], [2, 0], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [15, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [8, 0], [2, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 0], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [3, 1], [3, 1], [2, 1], [1, 1], [3, 1], [1, 1], [2, 1], [2, 1], [2, 1], [2, 1], [11, 1], [9, 0], [16, 1], [5, 1], [17, 1], [23, 1], [7, 1], [24, 1], [9, 1], [7, 1], [2, 1], [6, 1], [21, 1], [5, 1], [21, 1], [2, 1], [2, 1], [4, 1], [2, 1], [1, 1], [4, 1], [3, 1], [1, 1], [2, 1], [9, 1], [1, 1], [1, 1], [6, 1], [1, 1], [2, 1], [5, 0], [8, 0], [6, 1], [2, 1], [7, 0], [3, 0], [5, 1], [12, 0], [7, 0], [4, 0], [2, 0], [4, 0], [1, 1], [7, 0], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [3, 1], [1, 1], [8, 1], [7, 1], [2, 1], [27, 1], [7, 1], [7, 0], [1, 0], [6, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 0], [1, 1], [2, 0], [2, 1], [1, 1], [14, 1], [2, 1], [2, 1], [8, 1], [6, 1], [2, 1], [3, 1], [5, 1], [4, 1], [1, 1], [6, 1], [2, 1], [1, 1], [1, 1], [11, 1], [10, 1], [5, 1], [5, 1], [5, 1], [12, 0], [30, 0], [5, 0], [9, 0], [23, 1], [7, 1], [27, 0], [1, 0], [14, 1], [18, 0], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [5, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [2, 1], [3, 1], [1, 1], [5, 1], [1, 1], [2, 1], [2, 1], [1, 1], [2, 1], [3, 1], [12, 1], [20, 1], [6, 1], [20, 1], [12, 1], [3, 1], [3, 1], [3, 1], [7, 1], [4, 1], [18, 1], [11, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 0], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [3, 0], [6, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [3, 1], [3, 1], [1, 1], [1, 1], [12, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 1], [6, 1], [5, 1], [4, 1], [17, 1], [19, 1], [4, 1], [5, 1], [1, 1], [7, 1], [5, 1], [2, 1], [15, 1], [8, 1], [9, 1], [11, 1], [3, 1], [6, 1], [1, 1], [1, 1], [5, 1], [5, 1], [5, 1], [1, 1], [8, 1], [11, 1], [3, 1], [4, 1], [1, 1], [6, 1], [20, 1], [6, 0], [4, 1], [4, 1], [2, 0], [3, 1], [6, 1], [9, 0], [13, 1], [8, 1], [7, 1], [3, 1], [6, 0], [5, 0], [9, 0], [8, 1], [2, 1], [9, 0], [4, 0], [2, 1], [32, 1], [28, 1], [3, 1], [14, 0], [8, 1], [5, 1], [6, 1], [2, 1], [4, 1], [11, 0], [4, 1], [8, 0], [6, 1], [1, 1], [1, 1], [10, 1], [1, 1], [5, 0], [2, 0], [3, 1], [5, 1], [5, 1], [26, 1], [3, 0], [6, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 0], [1, 1], [10, 1], [1, 1], [2, 1], [1, 1], [10, 1], [27, 1], [1, 1], [10, 1], [2, 1], [4, 1], [1, 1], [2, 1], [1, 1], [5, 1], [4, 1], [4, 1], [5, 1], [3, 1], [7, 1], [5, 1], [1, 1], [5, 1], [5, 1], [6, 0], [8, 1], [10, 1], [19, 0], [1, 1], [11, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [9, 1], [2, 1], [4, 1], [2, 1], [6, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [1, 1], [3, 1], [15, 1], [2, 1], [2, 1], [4, 1], [10, 1], [13, 1], [6, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 0], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [1, 1], [2, 1], [3, 1], [2, 1], [1, 1], [1, 1], [7, 1], [1, 1], [1, 1], [2, 1], [5, 1], [8, 1], [5, 1], [28, 1], [1, 1], [5, 1], [3, 1], [1, 1], [3, 1], [3, 1], [22, 1], [1, 1], [19, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [21, 1], [9, 1], [2, 1], [1, 1], [4, 1], [6, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [3, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [16, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [3, 1], [1, 1], [7, 1], [1, 1], [1, 1], [4, 1], [4, 0], [4, 1], [2, 1], [15, 1], [10, 0], [2, 1], [3, 1], [1, 1], [4, 1], [2, 1], [4, 1], [1, 1], [2, 1], [6, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [5, 1], [3, 1], [1, 1], [1, 1], [2, 1], [2, 1], [2, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [2, 1], [2, 1], [9, 1], [19, 1], [4, 1], [26, 1], [8, 1], [6, 1], [1, 1], [7, 1], [29, 1], [17, 1], [11, 1], [14, 1], [1, 1], [2, 0], [1, 1], [1, 0], [1, 1], [3, 1], [1, 1], [1, 1], [5, 1], [6, 1], [2, 1], [3, 1], [11, 1], [2, 1], [3, 1], [6, 1], [7, 1], [7, 1], [2, 1], [4, 1], [4, 1], [6, 1], [10, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [11, 1], [1, 1], [2, 1], [6, 1], [5, 1], [1, 1], [4, 1], [1, 1], [4, 1], [1, 1], [10, 1], [1, 0], [2, 1], [6, 1], [1, 1], [2, 1], [6, 1], [4, 1], [1, 1], [13, 1], [19, 1], [2, 1], [5, 1], [4, 1], [7, 1], [2, 1], [5, 1], [3, 1], [18, 1], [6, 0], [8, 0], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [2, 1], [2, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [3, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [5, 1], [2, 1], [18, 1], [14, 1], [2, 1], [2, 1], [6, 1], [7, 1], [3, 1], [3, 1], [2, 1], [8, 1], [2, 1], [2, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [7, 1], [1, 1], [2, 1], [2, 1], [3, 1], [3, 1], [1, 1], [4, 1], [3, 1], [2, 1], [5, 1], [1, 1], [1, 1], [1, 1], [8, 1], [11, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [7, 1], [1, 1], [1, 1], [23, 1], [4, 1], [5, 1], [7, 1], [5, 1], [3, 1], [4, 0], [4, 1], [5, 0], [3, 1], [8, 0], [1, 1], [10, 1], [3, 0], [5, 1], [8, 0], [2, 1], [3, 1], [2, 1], [1, 1], [2, 1], [4, 1], [4, 0], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [2, 1], [2, 1], [1, 0], [1, 1], [1, 1], [2, 0], [2, 0], [2, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [3, 1], [4, 1], [3, 1], [2, 1], [3, 1], [2, 1], [3, 1], [9, 1], [2, 1], [3, 1], [3, 1], [2, 1], [1, 1], [10, 1], [3, 1], [10, 0], [2, 1], [2, 1], [1, 1], [3, 1], [3, 1], [1, 1], [2, 1], [2, 1], [8, 0], [1, 1], [1, 1], [6, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [3, 1], [4, 1], [4, 1], [4, 1], [1, 1], [4, 1], [6, 1], [7, 1], [11, 1], [2, 1], [5, 1], [2, 1], [3, 1], [4, 1], [3, 1], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [16, 1], [3, 1], [1, 1], [1, 1], [1, 1], [3, 1], [2, 1], [3, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [9, 1], [15, 1], [7, 1], [5, 1], [11, 1], [3, 1], [2, 1], [2, 1], [2, 1], [14, 1], [4, 1], [11, 1], [2, 1], [2, 1], [4, 1], [6, 1], [2, 1], [18, 1], [4, 1], [6, 1], [4, 1], [2, 1], [6, 1], [6, 1], [3, 1], [2, 1], [1, 1], [5, 1], [4, 1], [14, 1], [9, 1], [24, 1], [24, 1], [14, 1], [7, 1], [1, 1], [2, 1], [8, 1], [31, 1], [16, 1], [14, 1], [10, 1], [19, 1], [2, 1], [3, 1], [1, 1], [2, 1], [4, 1], [2, 1], [3, 1], [2, 1], [1, 1], [8, 1], [4, 1], [8, 1], [2, 1], [1, 1], [4, 1], [12, 1], [3, 1], [2, 1], [6, 1], [9, 0], [1, 1], [5, 1], [7, 1], [3, 1], [12, 1], [9, 1], [28, 1], [7, 1], [5, 1], [11, 0], [1, 1], [3, 1], [9, 1], [1, 1], [40, 1], [12, 1], [3, 0], [11, 1], [15, 0], [19, 0], [3, 1], [39, 0], [1, 0], [2, 0], [2, 1], [15, 0], [18, 1], [3, 1], [2, 1], [1, 1], [1, 1], [6, 1], [4, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [5, 1], [4, 1], [1, 1], [1, 1], [7, 1], [1, 1], [1, 1], [9, 1], [11, 1], [7, 1], [5, 1], [5, 1], [4, 1], [4, 1], [2, 1], [2, 1], [1, 1], [11, 1], [7, 1], [4, 1], [5, 1], [6, 1], [3, 1], [1, 1], [2, 1], [1, 1], [3, 1], [2, 1], [3, 1], [21, 1], [8, 1], [5, 1], [24, 1], [4, 1], [13, 1], [7, 1], [3, 1], [3, 1], [4, 1], [8, 1], [3, 1], [10, 1], [2, 1], [4, 1], [5, 1], [2, 1], [4, 1], [22, 0], [4, 1], [1, 1], [2, 1], [3, 1], [3, 1], [1, 1], [1, 1], [1, 1], [10, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [1, 1], [1, 1], [1, 0], [1, 1], [1, 0], [1, 1], [2, 1], [1, 1], [2, 1], [2, 1], [5, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [4, 1], [3, 1], [6, 1], [3, 1], [5, 1], [7, 1], [4, 1], [4, 1], [16, 1], [8, 1], [3, 1], [7, 1], [7, 1], [6, 1], [12, 1], [1, 1], [4, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [8, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [4, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [35, 1], [7, 1], [1, 1], [1, 1], [1, 1], [9, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 1], [14, 1], [4, 1], [3, 1], [5, 1], [4, 1], [4, 1], [3, 1], [2, 1], [4, 1], [2, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [4, 1], [4, 1], [18, 1], [10, 1], [6, 1], [4, 1], [7, 1], [2, 1], [2, 1], [8, 1], [11, 1], [9, 1], [11, 1], [9, 1], [5, 1], [2, 1], [4, 1], [1, 1], [1, 1], [2, 1], [11, 1], [3, 1], [12, 1], [8, 1], [14, 1], [6, 1], [32, 1], [15, 1], [9, 1], [19, 1], [6, 1], [8, 0], [23, 1], [33, 1], [6, 1], [2, 1], [1, 1], [3, 1], [1, 1], [4, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [9, 1], [8, 0], [11, 0], [4, 0], [5, 1], [8, 1], [11, 1], [5, 0], [8, 1], [6, 1], [1, 0], [3, 0], [1, 0], [4, 0], [1, 0], [2, 0], [1, 0], [5, 0], [4, 0], [1, 0], [11, 0], [1, 0], [3, 1], [10, 1], [10, 1], [7, 1], [3, 1], [4, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [2, 1], [4, 1], [3, 1], [7, 1], [9, 1], [1, 1], [8, 1], [3, 1], [1, 1], [5, 1], [2, 1], [8, 1], [1, 1], [3, 1], [3, 1], [3, 1], [2, 1], [1, 1], [2, 1], [1, 1], [9, 1], [5, 0], [9, 1], [6, 1], [7, 0], [7, 1], [1, 0], [10, 0], [6, 0], [3, 1], [21, 0], [1, 1], [4, 0], [1, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [4, 1], [2, 1], [1, 1], [1, 1], [3, 1], [3, 1], [3, 1], [3, 1], [1, 1], [1, 1], [3, 1], [5, 1], [1, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [3, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [4, 1], [2, 1], [2, 1], [2, 1], [2, 1], [2, 1], [4, 1], [3, 1], [3, 1], [8, 1], [3, 1], [5, 1], [3, 1], [4, 1], [11, 1], [7, 1], [2, 1], [5, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [7, 1], [1, 1], [2, 1], [9, 1], [6, 1], [2, 1], [5, 1], [16, 1], [2, 1], [3, 1], [8, 1], [4, 1], [2, 1], [5, 1], [1, 1], [9, 1], [17, 1], [11, 1], [7, 1], [5, 1], [32, 0], [8, 0], [4, 0], [6, 0], [2, 1], [2, 0], [2, 1], [1, 0], [1, 0], [7, 0], [13, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [7, 1], [3, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [6, 1], [3, 1], [1, 1], [4, 1], [7, 1], [2, 1], [1, 1], [2, 1], [2, 1], [1, 1], [2, 1], [3, 1], [1, 1], [1, 1], [1, 1], [1, 1], [12, 1], [5, 1], [2, 1], [14, 1], [10, 1], [14, 1], [5, 1], [1, 1], [5, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 0], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [2, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 0], [7, 1], [2, 1], [16, 0], [16, 1], [2, 0], [1, 1], [2, 1], [4, 1], [1, 1], [9, 1], [1, 1], [1, 1], [9, 1], [1, 1], [10, 1], [2, 1], [10, 1], [22, 0], [15, 1], [10, 0], [6, 0], [4, 1], [4, 0], [6, 1], [7, 1], [4, 0], [4, 0], [17, 0], [1, 1], [1, 1], [1, 1], [1, 1], [9, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [4, 0], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [3, 1], [2, 1], [7, 1], [1, 1], [3, 1], [28, 1], [2, 1], [2, 1], [17, 1], [1, 1], [1, 1], [4, 1], [1, 1], [3, 1], [2, 1], [1, 1], [1, 1], [1, 1], [3, 1], [1, 1]]\n"
+ ]
+ }
+ ],
+ "source": [
+ "#cargando conjunto de datos\n",
+ "archivoF19Early=open(\"data/F19/Test/early.csv\")\n",
+ "\n",
+ "archivoF19Early.readline()\n",
+ "\n",
+ "datos_x_test=[]\n",
+ "datos_y_test=[]\n",
+ "\n",
+ "\n",
+ "for linea in archivoF19Early:\n",
+ " \n",
+ " linea=linea.strip(\"\\n\").split(\",\")\n",
+ " \n",
+ " subjectID=linea[0]\n",
+ " assignmentID=linea[1]\n",
+ " problemID=linea[2]\n",
+ " attempts=int(linea[3])\n",
+ " correctEventually=linea[4]\n",
+ " label=linea[5]\n",
+ " \n",
+ " if(label==\"True\"):\n",
+ " label=1\n",
+ " else:\n",
+ " label=0\n",
+ " \n",
+ " if(correctEventually==\"True\"):\n",
+ " correctEventually=1\n",
+ " else:\n",
+ " correctEventually=0\n",
+ " \n",
+ " fila=[attempts,correctEventually]\n",
+ " \n",
+ " datos_x_test.append(fila)\n",
+ " \n",
+ " datos_y_test.append(label)\n",
+ "\n",
+ "archivoF19Early.close()\n",
+ "print(datos_x)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "id": "c8363b0a",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "C:\\Users\\Oculus\\miniconda3\\lib\\site-packages\\sklearn\\base.py:561: FutureWarning: Arrays of bytes/strings is being converted to decimal numbers if dtype='numeric'. This behavior is deprecated in 0.24 and will be removed in 1.1 (renaming of 0.26). Please convert your data to numeric values explicitly instead.\n",
+ " X = check_array(X, **check_params)\n"
+ ]
+ },
+ {
+ "ename": "ValueError",
+ "evalue": "Unable to convert array of bytes/strings into decimal numbers with dtype='numeric'",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)\u001b[0m\n\u001b[0;32m 778\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 779\u001b[1;33m \u001b[0marray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 780\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;31mValueError\u001b[0m: could not convert string to float: 'True'",
+ "\nThe above exception was the direct cause of the following exception:\n",
+ "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
+ "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_14864/561092016.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mprediction_y\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mregresionLogistica\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdatos_x_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\sklearn\\linear_model\\_base.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m 423\u001b[0m \u001b[0mPredicted\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mlabel\u001b[0m \u001b[0mper\u001b[0m \u001b[0msample\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 424\u001b[0m \"\"\"\n\u001b[1;32m--> 425\u001b[1;33m \u001b[0mscores\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecision_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 426\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 427\u001b[0m \u001b[0mindices\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mscores\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\sklearn\\linear_model\\_base.py\u001b[0m in \u001b[0;36mdecision_function\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m 405\u001b[0m \u001b[0mcheck_is_fitted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 406\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 407\u001b[1;33m \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_validate_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"csr\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreset\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 408\u001b[0m \u001b[0mscores\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msafe_sparse_dot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcoef_\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdense_output\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mintercept_\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 409\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mscores\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mscores\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mscores\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\sklearn\\base.py\u001b[0m in \u001b[0;36m_validate_data\u001b[1;34m(self, X, y, reset, validate_separately, **check_params)\u001b[0m\n\u001b[0;32m 559\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Validation should be done on X, y or both.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 560\u001b[0m \u001b[1;32melif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mno_val_X\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mno_val_y\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 561\u001b[1;33m \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mcheck_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 562\u001b[0m \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 563\u001b[0m \u001b[1;32melif\u001b[0m \u001b[0mno_val_X\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mno_val_y\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)\u001b[0m\n\u001b[0;32m 779\u001b[0m \u001b[0marray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 780\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 781\u001b[1;33m raise ValueError(\n\u001b[0m\u001b[0;32m 782\u001b[0m \u001b[1;34m\"Unable to convert array of bytes/strings \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 783\u001b[0m \u001b[1;34m\"into decimal numbers with dtype='numeric'\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;31mValueError\u001b[0m: Unable to convert array of bytes/strings into decimal numbers with dtype='numeric'"
+ ]
+ }
+ ],
+ "source": [
+ "prediction_y=regresionLogistica.predict(datos_x_test)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "id": "6dbe6f23",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[[1018 2543]\n",
+ " [ 0 0]]\n"
+ ]
+ }
+ ],
+ "source": [
+ "#verificando la matriz de confusión\n",
+ "from sklearn.metrics import confusion_matrix\n",
+ "matriz=confusion_matrix(datos_y_test,prediction_y)\n",
+ "print(matriz)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6a9d552f",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/naive_model.ipynb b/naive_model.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..f31f66b7ebc9c1abb88020a0aec7be0aa97294d8
--- /dev/null
+++ b/naive_model.ipynb
@@ -0,0 +1,1056 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "from ProgSnap2 import ProgSnap2Dataset\n",
+ "from ProgSnap2 import PS2\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.preprocessing import OneHotEncoder\n",
+ "import numpy as np\n",
+ "import os\n",
+ "from os import path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "semester = 'S19'\n",
+ "BASE_PATH = os.path.join('data', 'Release', semester)\n",
+ "TRAIN_PATH = os.path.join(BASE_PATH, 'Train')\n",
+ "TEST_PATH = os.path.join(BASE_PATH, 'Test')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_ps2 = ProgSnap2Dataset(os.path.join(TRAIN_PATH, 'Data')) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Attempts</th>\n",
+ " <th>CorrectEventually</th>\n",
+ " <th>Label</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>3</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>5</td>\n",
+ " <td>3</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>12</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>13</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Attempts \\\n",
+ "0 04c32d4d95425f73b3a1d6502aed4d48 439.0 1 1 \n",
+ "1 04c32d4d95425f73b3a1d6502aed4d48 439.0 3 2 \n",
+ "2 04c32d4d95425f73b3a1d6502aed4d48 439.0 5 3 \n",
+ "3 04c32d4d95425f73b3a1d6502aed4d48 439.0 12 1 \n",
+ "4 04c32d4d95425f73b3a1d6502aed4d48 439.0 13 2 \n",
+ "\n",
+ " CorrectEventually Label \n",
+ "0 True True \n",
+ "1 True True \n",
+ "2 True True \n",
+ "3 True True \n",
+ "4 True True "
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# The early dataset will help us to feature extraction,\n",
+ "# but we're not actually predicting anything here\n",
+ "# Note: we could still use this for model training if desired.\n",
+ "early_train = pd.read_csv(os.path.join(TRAIN_PATH, 'early.csv'))\n",
+ "early_train.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Label</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>41</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>43</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>44</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>46</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>49</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Label\n",
+ "0 04c32d4d95425f73b3a1d6502aed4d48 494.0 41 False\n",
+ "1 04c32d4d95425f73b3a1d6502aed4d48 494.0 43 True\n",
+ "2 04c32d4d95425f73b3a1d6502aed4d48 494.0 44 True\n",
+ "3 04c32d4d95425f73b3a1d6502aed4d48 494.0 46 True\n",
+ "4 04c32d4d95425f73b3a1d6502aed4d48 494.0 49 True"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# The late dataset contains the problems that we're actually predicting for.\n",
+ "# The training portion of it includes labels.\n",
+ "late_train = pd.read_csv(os.path.join(TRAIN_PATH, 'late.csv'))\n",
+ "late_train.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "X_train_base = late_train.copy().drop('Label', axis=1)\n",
+ "y_train = late_train['Label'].values"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "problem_encoder = OneHotEncoder().fit(X_train_base[PS2.ProblemID].values.reshape(-1, 1))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([[1., 0., 0., ..., 0., 0., 0.],\n",
+ " [0., 1., 0., ..., 0., 0., 0.],\n",
+ " [0., 0., 1., ..., 0., 0., 0.],\n",
+ " ...,\n",
+ " [0., 0., 0., ..., 0., 0., 0.],\n",
+ " [0., 0., 0., ..., 0., 1., 0.],\n",
+ " [0., 0., 0., ..., 0., 0., 1.]])"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "problem_encoder.transform(X_train_base[PS2.ProblemID].values.reshape(-1, 1)).toarray()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Task 1\n",
+ "\n",
+ "In this task, we do per-problem prediction, extracting features from performance on the 30 early problems for a given student to predict performance on each of 20 later problems. Our model should, in effect, learn the releationship between the knowledge practiced in these problems (though our naive example here won't get that far)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def extract_instance_features(instance, early_df):\n",
+ " instance = instance.copy()\n",
+ " subject_id = instance[PS2.SubjectID]\n",
+ " early_problems = early_df[early_df[PS2.SubjectID] == subject_id]\n",
+ " # Extract very naive features about the student\n",
+ " # (without respect to the problem bring predicted)\n",
+ " # Number of early problems attempted\n",
+ " instance['ProblemsAttempted'] = early_problems.shape[0]\n",
+ " # Percentage of early problems gotten correct eventually\n",
+ " instance['PercCorrectEventually'] = np.mean(early_problems['CorrectEventually'])\n",
+ " # Median attempts made on early problems\n",
+ " instance['MedAttempts'] = np.median(early_problems['Attempts'])\n",
+ " # Max attempts made on early problems\n",
+ " instance['MaxAttempts'] = np.max(early_problems['Attempts'])\n",
+ " # Percentage of problems gotten correct on the first try\n",
+ " instance['PercCorrectFirstTry'] = np.mean(early_problems['Attempts'] == 1)\n",
+ " #instance = instance.drop('AssignmentID')\n",
+ " #instance = instance.drop('ProblemID')\n",
+ " instance = instance.drop('SubjectID')\n",
+ " return instance"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "SubjectID 04c32d4d95425f73b3a1d6502aed4d48\n",
+ "AssignmentID 494.0\n",
+ "ProblemID 41\n",
+ "Name: 0, dtype: object"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "X_train_base.iloc[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "AssignmentID 494.0\n",
+ "ProblemID 41\n",
+ "ProblemsAttempted 30\n",
+ "PercCorrectEventually 1.0\n",
+ "MedAttempts 6.5\n",
+ "MaxAttempts 45\n",
+ "PercCorrectFirstTry 0.166667\n",
+ "Name: 0, dtype: object"
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "extract_instance_features(X_train_base.iloc[0], early_train)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def extract_features(X, early_df, scaler, is_train):\n",
+ " # First extract performance features for each row\n",
+ " features = X.apply(lambda instance: extract_instance_features(instance, early_df), axis=1)\n",
+ " # Then one-hot encode the problem_id and append it\n",
+ " problem_ids = problem_encoder.transform(features[PS2.ProblemID].values.reshape(-1, 1)).toarray()\n",
+ " # Then get rid of nominal features\n",
+ " features.drop([PS2.AssignmentID, PS2.ProblemID], axis=1, inplace=True)\n",
+ " # Then scale the continuous features, fitting the scaler if this is training\n",
+ " if is_train:\n",
+ " scaler.fit(features)\n",
+ " features = scaler.transform(features)\n",
+ " \n",
+ " # Return continuous and one-hot features together\n",
+ " return np.concatenate([features, problem_ids], axis=1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.preprocessing import StandardScaler\n",
+ "scaler = StandardScaler()\n",
+ "X_train = extract_features(X_train_base, early_train, scaler, True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4201, 25)\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "array([[ 0.51751812, 0.58371895, 1.76922077, 1.70602676, -0.89569333,\n",
+ " 1. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ,\n",
+ " 0. , 0. , 0. , 0. , 0. ]])"
+ ]
+ },
+ "execution_count": 38,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "print(X_train.shape)\n",
+ "X_train[:1,]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Evaluate the Training Performance of the Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 39,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4201,)\n"
+ ]
+ }
+ ],
+ "source": [
+ "from sklearn.linear_model import LogisticRegressionCV\n",
+ "\n",
+ "model = LogisticRegressionCV()\n",
+ "model.fit(X_train, y_train) #entrenamos el modelo\n",
+ "train_predictions = model.predict(X_train) #testeamos el modelo\n",
+ "print(train_predictions.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'y_train' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)",
+ "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_16304/2480459126.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mf1_score\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclassification_report\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_predictions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 6\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'AUC: '\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mroc_auc_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_predictions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 7\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Macro F1: '\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf1_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_predictions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maverage\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'macro'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;31mNameError\u001b[0m: name 'y_train' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "from sklearn.metrics import classification_report\n",
+ "from sklearn.metrics import roc_auc_score\n",
+ "from sklearn.metrics import f1_score\n",
+ "\n",
+ "print(classification_report(y_train, train_predictions))\n",
+ "print('AUC: ' + str(roc_auc_score(y_train, train_predictions)))\n",
+ "print('Macro F1: ' + str(f1_score(y_train, train_predictions, average='macro')))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.metrics import plot_roc_curve\n",
+ "\n",
+ "plot_roc_curve(model, X_train, y_train)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Evaluate the CV Performance of the Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.model_selection import cross_validate\n",
+ "\n",
+ "model = LogisticRegressionCV()\n",
+ "cv_results = cross_validate(model, X_train, y_train, cv=10, scoring=['accuracy', 'f1_macro', 'roc_auc'])\n",
+ "print(f'Accuracy: {np.mean(cv_results[\"test_accuracy\"])}')\n",
+ "print(f'AUC: {np.mean(cv_results[\"test_roc_auc\"])}')\n",
+ "print(f'Macro F1: {np.mean(cv_results[\"test_f1_macro\"])}')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Predict on the test data (S19)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "early_test = pd.read_csv(os.path.join(TEST_PATH, 'early.csv'))\n",
+ "late_test = pd.read_csv(os.path.join(TEST_PATH, 'late.csv'))\n",
+ "X_test = extract_features(late_test, early_test, scaler, False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "X_test.shape"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "When writing output to be judged, make _sure_ to **output probabilities** for the positive class, so that we can calculate AUC when judging!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = LogisticRegressionCV()\n",
+ "model.fit(X_train, y_train)\n",
+ "# Note the use of predict_proba (the [:,1] gets the positive probabilities)\n",
+ "predictions = model.predict_proba(X_test)[:,1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "predictions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "predictions_df = late_test.copy()\n",
+ "predictions_df['Label'] = predictions\n",
+ "predictions_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# We don't have the test labels - you have to submit to evaluate it"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# We use res/predictions.csv, since that's where the scoring program expects it\n",
+ "# but you can change this directory\n",
+ "path = os.path.join('data', 'Prediction', semester, 'basic_LR_task1', 'res')\n",
+ "os.makedirs(path, exist_ok=True)\n",
+ "predictions_df.to_csv(os.path.join(path, 'predictions.csv'), index=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Predict on the test data for the _next_ semester (F19)\n",
+ "\n",
+ "Here we see if our model will still be useful next semester. Again, we don't have labels for this test dataset, so we'll have to submit to see how well we did.\n",
+ "\n",
+ "Later in the year, there will be _training_ data released for F19, which will be tested on a different track (not cross-semester)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 40,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "F19_TEST_PATH = os.path.join('data', 'Release', 'F19', 'Test')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "early_test = pd.read_csv(os.path.join(F19_TEST_PATH, 'early.csv'))\n",
+ "late_test = pd.read_csv(os.path.join(F19_TEST_PATH, 'late.csv'))\n",
+ "X_test = extract_features(late_test, early_test, scaler, False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(2365, 25)"
+ ]
+ },
+ "execution_count": 42,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "X_test.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = LogisticRegressionCV()\n",
+ "model.fit(X_train, y_train)\n",
+ "predictions = model.predict_proba(X_test)[:,1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([0.81344772, 0.74948573, 0.87591233, ..., 0.40992631, 0.37947527,\n",
+ " 0.35654374])"
+ ]
+ },
+ "execution_count": 45,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "predictions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "predictions_df = late_test.copy()\n",
+ "predictions_df['Label'] = predictions\n",
+ "predictions_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# We don't have the test labels - you have to submit to evaluate it"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "path = os.path.join('data', 'Prediction', 'F19', 'basic_LR_task1', 'res')\n",
+ "os.makedirs(path, exist_ok=True)\n",
+ "predictions_df.to_csv(os.path.join(path, 'predictions.csv'), index=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Task 2\n",
+ "\n",
+ "In this task, we are predicting final performance, extracting features from performance on the 30 early problems for a given student to predict the final exam grade."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "train_base = train_ps2.load_link_table('Subject')\n",
+ "train_base"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_base = late_train.groupby('SubjectID')['Label'].sum().to_frame('X-Grade').reset_index()\n",
+ "train_base['X-Grade'] = train_base['X-Grade'] / 20\n",
+ "train_base"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "y_train = train_base['X-Grade']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# The base of our X dataset is just a set of SubjectIDs\n",
+ "X_train_base = train_base.drop('X-Grade', axis=1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.preprocessing import StandardScaler\n",
+ "X_train_unscaled = X_train_base.apply(lambda row: extract_instance_features(row, early_train), axis=1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "X_train_unscaled"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "scaler = StandardScaler()\n",
+ "X_train = scaler.fit_transform(X_train_unscaled)\n",
+ "X_train"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "X_train_unscaled.join(train_base).drop(PS2.SubjectID, axis=1).corr()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Evaluate the Training Performance of the Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.linear_model import LassoCV\n",
+ "from sklearn.linear_model import LinearRegression\n",
+ "\n",
+ "model = LassoCV()\n",
+ "model.fit(X_train, y_train)\n",
+ "train_predictions = model.predict(X_train)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "plt.scatter(y_train, train_predictions)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.metrics import mean_squared_error\n",
+ "from sklearn.metrics import r2_score\n",
+ "\n",
+ "print('Training MSE: ' + str(mean_squared_error(y_train, train_predictions)))\n",
+ "print('Training R2: ' + str(r2_score(y_train, train_predictions)))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Evaluate the CV Performance of the Model\n",
+ "\n",
+ "The naive model performs quite poorly evaluated by CV on the training data (R2 of 0.083). Can you improve on it?"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "from sklearn.model_selection import cross_validate\n",
+ "\n",
+ "model = LassoCV()\n",
+ "cv_results = cross_validate(model, X_train, y_train, cv=10, scoring=['neg_mean_squared_error', 'r2'])\n",
+ "print(f'MSE: {-np.mean(cv_results[\"test_neg_mean_squared_error\"])}')\n",
+ "print(f'R2: {np.mean(cv_results[\"test_r2\"])}')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Predict on the test data (S19)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "test_ps2 = ProgSnap2Dataset(os.path.join(TEST_PATH, 'Data'))\n",
+ "early_test = pd.read_csv(os.path.join(TEST_PATH, 'early.csv'))\n",
+ "late_test = pd.read_csv(os.path.join(TEST_PATH, 'late.csv'))\n",
+ "\n",
+ "# The Subject.csv link table is just the SubjectIDs to be predicted\n",
+ "X_test_base = test_ps2.load_link_table('Subject')\n",
+ "X_test_unscaled = X_test_base.apply(lambda row: extract_instance_features(row, early_test), axis=1)\n",
+ "X_test_unscaled"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "X_test = scaler.transform(X_test_unscaled)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "X_test.shape"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "When writing output to be judged, make _sure_ to **output probabilities** for the positive class, so that we can calculate AUC when judging!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = LassoCV()\n",
+ "model.fit(X_train, y_train)\n",
+ "predictions = model.predict(X_test)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "predictions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "predictions_df = X_test_base.copy()\n",
+ "predictions_df['X-Grade'] = predictions\n",
+ "predictions_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# We don't have the test labels - you have to submit to evaluate it"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# We use res/predictions.csv, since that's where the scoring rogram expects it\n",
+ "# but you can change this directory\n",
+ "path = os.path.join('data', 'Prediction', semester, 'basic_LR_task2', 'res')\n",
+ "os.makedirs(path, exist_ok=True)\n",
+ "predictions_df.to_csv(os.path.join(path, 'predictions.csv'), index=False)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/preprocess.ipynb b/preprocess.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..0181048a9ef1073efa9efc3d8485a856d4381345
--- /dev/null
+++ b/preprocess.ipynb
@@ -0,0 +1,2921 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "from ProgSnap2 import ProgSnap2Dataset\n",
+ "from ProgSnap2 import PS2\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "import numpy as np\n",
+ "import os\n",
+ "from os import path"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Loading and Cleaning Data\n",
+ "We load our data using the ProgSnap2Dataset class. This comes with both a main event table and a LinkTable giving students final exam data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "semester = 'S19'\n",
+ "PATH = \"data/CodeWorkout/\" + semester+\"/Train\"+\"/Data\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data = ProgSnap2Dataset(PATH)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "KeyError",
+ "evalue": "\"['Attempt'] not found in axis\"",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)",
+ "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_12600/3379611682.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[1;31m# Drop the attempt column, since it's calculated incorrectly\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop_main_table_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Attempt'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
+ "\u001b[1;32m~\\Documents\\Ing. Juan Vera\\retoIA\\proyectos\\CSEDMDataChallenge\\ProgSnap2.py\u001b[0m in \u001b[0;36mdrop_main_table_column\u001b[1;34m(self, column)\u001b[0m\n\u001b[0;32m 130\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdrop_main_table_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumn\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 131\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_main_table\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 132\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmain_table\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 133\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 134\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0msave_subset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmain_table_filterer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy_link_tables\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\util\\_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m 309\u001b[0m \u001b[0mstacklevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 310\u001b[0m )\n\u001b[1;32m--> 311\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 312\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 313\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36mdrop\u001b[1;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[0;32m 4904\u001b[0m \u001b[0mweight\u001b[0m \u001b[1;36m1.0\u001b[0m \u001b[1;36m0.8\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4905\u001b[0m \"\"\"\n\u001b[1;32m-> 4906\u001b[1;33m return super().drop(\n\u001b[0m\u001b[0;32m 4907\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4908\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36mdrop\u001b[1;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[0;32m 4148\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[1;32min\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4149\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mlabels\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4150\u001b[1;33m \u001b[0mobj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_drop_axis\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 4151\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4152\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36m_drop_axis\u001b[1;34m(self, labels, axis, level, errors)\u001b[0m\n\u001b[0;32m 4183\u001b[0m \u001b[0mnew_axis\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4184\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4185\u001b[1;33m \u001b[0mnew_axis\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 4186\u001b[0m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreindex\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0maxis_name\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mnew_axis\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4187\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mdrop\u001b[1;34m(self, labels, errors)\u001b[0m\n\u001b[0;32m 6015\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0many\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 6016\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0merrors\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m\"ignore\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 6017\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mf\"{labels[mask]} not found in axis\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 6018\u001b[0m \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mindexer\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m~\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 6019\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;31mKeyError\u001b[0m: \"['Attempt'] not found in axis\""
+ ]
+ }
+ ],
+ "source": [
+ "# Drop the attempt column, since it's calculated incorrectly\n",
+ "data.drop_main_table_column('Attempt')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "main_table = data.get_main_table()\n",
+ "student_table = data.load_link_table('Subject')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>Order</th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>ToolInstances</th>\n",
+ " <th>ServerTimestamp</th>\n",
+ " <th>ServerTimezone</th>\n",
+ " <th>CourseID</th>\n",
+ " <th>CourseSectionID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>CodeStateID</th>\n",
+ " <th>IsEventOrderingConsistent</th>\n",
+ " <th>EventType</th>\n",
+ " <th>Score</th>\n",
+ " <th>Compile.Result</th>\n",
+ " <th>CompileMessageType</th>\n",
+ " <th>CompileMessageData</th>\n",
+ " <th>EventID</th>\n",
+ " <th>ParentEventID</th>\n",
+ " <th>SourceLocation</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>119441</td>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>Java 8; CodeWorkout</td>\n",
+ " <td>2019-02-23T22:44:51</td>\n",
+ " <td>UTC</td>\n",
+ " <td>CS 1</td>\n",
+ " <td>2</td>\n",
+ " <td>439.0</td>\n",
+ " <td>1</td>\n",
+ " <td>4531059d41ba170482b4e43d4d94d857c0e45dbb</td>\n",
+ " <td>True</td>\n",
+ " <td>Run.Program</td>\n",
+ " <td>1.0000</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " <td>1-68976</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>119442</td>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>Java 8; CodeWorkout</td>\n",
+ " <td>2019-02-23T22:44:51</td>\n",
+ " <td>UTC</td>\n",
+ " <td>CS 1</td>\n",
+ " <td>2</td>\n",
+ " <td>439.0</td>\n",
+ " <td>1</td>\n",
+ " <td>4531059d41ba170482b4e43d4d94d857c0e45dbb</td>\n",
+ " <td>True</td>\n",
+ " <td>Compile</td>\n",
+ " <td>NaN</td>\n",
+ " <td>Success</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " <td>1-68977</td>\n",
+ " <td>1-68976</td>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>134115</td>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>Java 8; CodeWorkout</td>\n",
+ " <td>2019-02-23T22:49:34</td>\n",
+ " <td>0</td>\n",
+ " <td>CS 1</td>\n",
+ " <td>2</td>\n",
+ " <td>439.0</td>\n",
+ " <td>3</td>\n",
+ " <td>69089e4182ecddd4b48c39c86c8ae2edb337b07c</td>\n",
+ " <td>True</td>\n",
+ " <td>Run.Program</td>\n",
+ " <td>0.8125</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " <td>3-67872</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>134116</td>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>Java 8; CodeWorkout</td>\n",
+ " <td>2019-02-23T22:49:34</td>\n",
+ " <td>0</td>\n",
+ " <td>CS 1</td>\n",
+ " <td>2</td>\n",
+ " <td>439.0</td>\n",
+ " <td>3</td>\n",
+ " <td>69089e4182ecddd4b48c39c86c8ae2edb337b07c</td>\n",
+ " <td>True</td>\n",
+ " <td>Compile</td>\n",
+ " <td>NaN</td>\n",
+ " <td>Success</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " <td>3-67873</td>\n",
+ " <td>3-67872</td>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>134117</td>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>Java 8; CodeWorkout</td>\n",
+ " <td>2019-02-23T22:50:47</td>\n",
+ " <td>0</td>\n",
+ " <td>CS 1</td>\n",
+ " <td>2</td>\n",
+ " <td>439.0</td>\n",
+ " <td>3</td>\n",
+ " <td>d565ccacd2e63b9414077ff2b4888622e37b80c6</td>\n",
+ " <td>True</td>\n",
+ " <td>Run.Program</td>\n",
+ " <td>1.0000</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " <td>3-67874</td>\n",
+ " <td>NaN</td>\n",
+ " <td>NaN</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " Order SubjectID ToolInstances \\\n",
+ "0 119441 04c32d4d95425f73b3a1d6502aed4d48 Java 8; CodeWorkout \n",
+ "1 119442 04c32d4d95425f73b3a1d6502aed4d48 Java 8; CodeWorkout \n",
+ "2 134115 04c32d4d95425f73b3a1d6502aed4d48 Java 8; CodeWorkout \n",
+ "3 134116 04c32d4d95425f73b3a1d6502aed4d48 Java 8; CodeWorkout \n",
+ "4 134117 04c32d4d95425f73b3a1d6502aed4d48 Java 8; CodeWorkout \n",
+ "\n",
+ " ServerTimestamp ServerTimezone CourseID CourseSectionID AssignmentID \\\n",
+ "0 2019-02-23T22:44:51 UTC CS 1 2 439.0 \n",
+ "1 2019-02-23T22:44:51 UTC CS 1 2 439.0 \n",
+ "2 2019-02-23T22:49:34 0 CS 1 2 439.0 \n",
+ "3 2019-02-23T22:49:34 0 CS 1 2 439.0 \n",
+ "4 2019-02-23T22:50:47 0 CS 1 2 439.0 \n",
+ "\n",
+ " ProblemID CodeStateID \\\n",
+ "0 1 4531059d41ba170482b4e43d4d94d857c0e45dbb \n",
+ "1 1 4531059d41ba170482b4e43d4d94d857c0e45dbb \n",
+ "2 3 69089e4182ecddd4b48c39c86c8ae2edb337b07c \n",
+ "3 3 69089e4182ecddd4b48c39c86c8ae2edb337b07c \n",
+ "4 3 d565ccacd2e63b9414077ff2b4888622e37b80c6 \n",
+ "\n",
+ " IsEventOrderingConsistent EventType Score Compile.Result \\\n",
+ "0 True Run.Program 1.0000 NaN \n",
+ "1 True Compile NaN Success \n",
+ "2 True Run.Program 0.8125 NaN \n",
+ "3 True Compile NaN Success \n",
+ "4 True Run.Program 1.0000 NaN \n",
+ "\n",
+ " CompileMessageType CompileMessageData EventID ParentEventID SourceLocation \n",
+ "0 NaN NaN 1-68976 NaN NaN \n",
+ "1 NaN NaN 1-68977 1-68976 NaN \n",
+ "2 NaN NaN 3-67872 NaN NaN \n",
+ "3 NaN NaN 3-67873 3-67872 NaN \n",
+ "4 NaN NaN 3-67874 NaN NaN "
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "main_table.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Fall 2019 Preprocessing\n",
+ "\n",
+ "There were some differences between F19 and S19:\n",
+ "* In F19 there was an additional assignment (between Assignment 4 and 5), which only ~70% of students completed, likely additional optional practice. We will not use this assignment for prediction, since it is abnormal and not in S19. Since it comes in between the two assignments we are using for prediction, we simply remove it.\n",
+ "* In F19 the AssignmentIDs were renamed, so we will update their names\n",
+ "* In F19 2 ProblemIDs were renamed (though the solutions were unchanged), so we will update their names"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "5\n",
+ "50\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(len(main_table[PS2.AssignmentID].unique()))\n",
+ "print(len(main_table[PS2.ProblemID].unique()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# This assignment has no analogue, but we use 500 to put it between the other 2\n",
+ "NEW_F19_ASSIGNMENT = 500"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if semester == 'F19':\n",
+ " assignment_map = {\n",
+ " 597: 439,\n",
+ " 600: 487,\n",
+ " 609: 492,\n",
+ " 615: 494,\n",
+ " 622: NEW_F19_ASSIGNMENT,\n",
+ " 631: 502,\n",
+ " }\n",
+ " print(np.mean(main_table[PS2.AssignmentID].isin(assignment_map)))\n",
+ " main_table[PS2.AssignmentID] = main_table[PS2.AssignmentID].map(assignment_map)\n",
+ " \n",
+ " # Two problems were renamed but are equivalent\n",
+ " problem_map = {problem_id: problem_id for problem_id in main_table[PS2.ProblemID].unique()}\n",
+ " problem_map[736] = 45\n",
+ " problem_map[737] = 48\n",
+ " print(np.mean(main_table[PS2.ProblemID].isin(problem_map)))\n",
+ " main_table[PS2.ProblemID] = main_table[PS2.ProblemID].map(problem_map, na_action='ignore')\n",
+ " \n",
+ " # Overwrite the main table so this is the one that's copied\n",
+ " data.set_main_table(main_table)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "5\n",
+ "50\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(len(main_table[PS2.AssignmentID].unique()))\n",
+ "print(len(main_table[PS2.ProblemID].unique()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0 1\n",
+ "1 1\n",
+ "2 3\n",
+ "3 3\n",
+ "4 3\n",
+ " ... \n",
+ "134503 71\n",
+ "134504 112\n",
+ "134505 112\n",
+ "134506 118\n",
+ "134507 118\n",
+ "Name: ProblemID, Length: 134508, dtype: int64"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "main_table[PS2.ProblemID]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Filtering Students\n",
+ "\n",
+ "Here we remove studens who did not take the final exam, since we cannot use these for Task 2 (final exam score prediction). While this does somewhat bias the dataset for Task 1, it also ensures a consistent set of training/testing students for both tasks.\n",
+ "\n",
+ "We can also see that few students are actually removed this way (381 -> 348 for S19)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "247\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "246"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Get the SubjectIDs where the final grade is non-0\n",
+ "# A 0 grade indicates the student did not take the final\n",
+ "print(len(student_table.index))\n",
+ "subject_ids = set(student_table[student_table['X-Grade'] != 0][PS2.SubjectID].unique())\n",
+ "subject_ids = subject_ids.intersection(set(student_table['SubjectID'].unique()))\n",
+ "len(subject_ids)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### EDA\n",
+ "\n",
+ "We want to confirm that our selected students have a good and well-distributed number of attempts at all the problems in the dataset, and the most problems were well-attempted. The stats and figures below suggest that this is the case: most problems are attempted by ~300/350 students, and most students complete ~40/50 problems."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "count 246.000000\n",
+ "mean 44.077236\n",
+ "std 8.137475\n",
+ "min 13.000000\n",
+ "25% 40.000000\n",
+ "50% 49.500000\n",
+ "75% 50.000000\n",
+ "max 50.000000\n",
+ "dtype: float64"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# First we check how many problems each student attempted\n",
+ "main_table_filtered = main_table[main_table[PS2.SubjectID].isin(subject_ids)]\n",
+ "problems_per_student = main_table_filtered.groupby(by=['SubjectID']).apply(lambda rows: len(rows[PS2.ProblemID].unique()))\n",
+ "# 75% of problems were attempted by at least 40 studens, so that's good\n",
+ "problems_per_student.describe()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Only 3 attempted fewer than 10 problems\n",
+ "sum(problems_per_student < 10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "count 50.000000\n",
+ "mean 216.860000\n",
+ "std 9.856128\n",
+ "min 195.000000\n",
+ "25% 209.000000\n",
+ "50% 216.500000\n",
+ "75% 226.000000\n",
+ "max 233.000000\n",
+ "dtype: float64"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Next we see how many students attempted each problem\n",
+ "students_per_problem = main_table_filtered.groupby(by=['AssignmentID', 'ProblemID']).apply(lambda rows: len(rows[PS2.SubjectID].unique()))\n",
+ "students_per_problem.describe()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<BarContainer object of 50 artists>"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAANOUlEQVR4nO3df6zd9V3H8eeLgmi2JYIUJICWmWoGxnVJgyTsDzZ01LkIRjElcekfmPoHRJawKOwf1KTJ/nDTmIhJFUITN7CGIWQxuqZuwSVm7KIolII0gFBb2040w38wwNs/7rfhcHsv98c5p/fe930+kuac8znfc87nk6bP8+Vzzz2kqpAk9XLOak9AkjR5xl2SGjLuktSQcZekhoy7JDV07mpPAOCiiy6qLVu2rPY0JGldeeqpp75XVZvnu29NxH3Lli3MzMys9jQkaV1J8u8L3ee2jCQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDW0Jn5Dda37+XNuOWPswDt/tQoz0Wr48B9/6Yyxl37rrlV/Lun9GPcxGH1Ja5Vxl1bIM3qtZe65S1JDxl2SGnJb5ixaaI/evfu1wa0RddIi7nPjuFgYpx3TST7/ctemtWHuG4VvEjrbWsR9IZ4R9+KZtbR0reO+XL4ZSOrCuGteH/+VPzhj7NuPfH4VZrJynulrI/PTMpLUkGfuWvc6n6HvevK2M8b2XXP/KsxE641n7pLUkGfu0jq00Bm9Z/o6zbhLWlPe+c+fPGPsnB/9twXHNT/jrqnq8Kmbjcj/Alj/jLu0AbiNs/EYd62auWf1Z+uMvvOna6TTjLukdc29+PkZdy2Le+jS+mDcJU3Vau3rb/QzeuO+Ti33DHo9nXGvp7lKa5Vxb2a1wmiQtV5slDN64y5pWeZus2yEj06uxzcE4y5pIpa7t+5n7KfLuG9wbqdI75p7hr7Wz87fz6LfCpnkiiTfTHI4yaEkdw7jFyY5kOTF4fKCkcfck+RIkheS3DjNBUiSzrSUr/x9C7irqj4CXAvcnuQq4G7gYFVtBQ4Otxnu2wlcDewA7kuyaRqTlyTNb9Ftmao6Dhwfrr+R5DBwGXATcP1w2D7gW8DvDOMPV9WbwMtJjgDXAP846clL0npyNn8wu6z/WUeSLcDHgO8AlwzhP/0GcPFw2GXAayMPOzqMzX2u3UlmksycOnVqBVOXJC1kyXFP8kHgEeBzVfX99zt0nrE6Y6Bqb1Vtr6rtmzdvXuo0JElLsKRPyyQ5j9mwf6WqvjYMn0hyaVUdT3IpcHIYPwpcMfLwy4Fjk5qwJK0Va/nz70v5tEyA+4HDVfXlkbseB3YN13cBj42M70xyfpIrga3Ak5ObsiRpMUs5c78O+CzwTJKnh7EvAF8E9ie5DXgVuAWgqg4l2Q88x+wnbW6vqrcnPXFJWqvWwhn9Uj4t823m30cHuGGBx+wB9owxL0nSGJb1aRlJ0vpg3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDS0a9yQPJDmZ5NmRsd9N8h9Jnh7+fHrkvnuSHEnyQpIbpzVxSdLClnLm/iCwY57xP6yqbcOfvwFIchWwE7h6eMx9STZNarKSpKVZNO5V9QTw+hKf7ybg4ap6s6peBo4A14wxP0nSCoyz535Hkn8dtm0uGMYuA14bOeboMHaGJLuTzCSZOXXq1BjTkCTNtdK4/ynwE8A24DjwpWE88xxb8z1BVe2tqu1VtX3z5s0rnIYkaT4rintVnaiqt6vqHeDPeHfr5ShwxcihlwPHxpuiJGm5VhT3JJeO3Pxl4PQnaR4HdiY5P8mVwFbgyfGmKElarnMXOyDJQ8D1wEVJjgL3Atcn2cbslssrwG8CVNWhJPuB54C3gNur6u2pzFyStKBF415Vt84zfP/7HL8H2DPOpCRJ4/E3VCWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ4vGPckDSU4meXZk7MIkB5K8OFxeMHLfPUmOJHkhyY3TmrgkaWFLOXN/ENgxZ+xu4GBVbQUODrdJchWwE7h6eMx9STZNbLaSpCVZNO5V9QTw+pzhm4B9w/V9wM0j4w9X1ZtV9TJwBLhmMlOVJC3VSvfcL6mq4wDD5cXD+GXAayPHHR3GzpBkd5KZJDOnTp1a4TQkSfOZ9A9UM89YzXdgVe2tqu1VtX3z5s0TnoYkbWwrjfuJJJcCDJcnh/GjwBUjx10OHFv59CRJK7HSuD8O7Bqu7wIeGxnfmeT8JFcCW4Enx5uiJGm5zl3sgCQPAdcDFyU5CtwLfBHYn+Q24FXgFoCqOpRkP/Ac8BZwe1W9PaW5S5IWsGjcq+rWBe66YYHj9wB7xpmUJGk8/oaqJDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDZ07zoOTvAK8AbwNvFVV25NcCPwlsAV4Bfi1qvrv8aYpSVqOSZy5f6KqtlXV9uH23cDBqtoKHBxuS5LOomlsy9wE7Buu7wNunsJrSJLex7hxL+AbSZ5KsnsYu6SqjgMMlxfP98Aku5PMJJk5derUmNOQJI0aa88duK6qjiW5GDiQ5PmlPrCq9gJ7AbZv315jzkOSNGKsM/eqOjZcngQeBa4BTiS5FGC4PDnuJCVJy7PiuCf5QJIPnb4OfAp4Fngc2DUctgt4bNxJSpKWZ5xtmUuAR5Ocfp6vVtXfJvkusD/JbcCrwC3jT1OStBwrjntVvQR8dJ7x/wJuGGdSkqTx+BuqktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJamhqcU9yY4kLyQ5kuTuab2OJOlMU4l7kk3AnwC/AFwF3Jrkqmm8liTpTNM6c78GOFJVL1XV/wEPAzdN6bUkSXOkqib/pMmvAjuq6jeG258Ffraq7hg5Zjewe7j5U8ALE3jpi4DvTeB51hPXvDG45o1jOev+8araPN8d505uPu+Recbe8y5SVXuBvRN90WSmqrZP8jnXOte8MbjmjWNS657WtsxR4IqR25cDx6b0WpKkOaYV9+8CW5NcmeQHgJ3A41N6LUnSHFPZlqmqt5LcAfwdsAl4oKoOTeO15pjoNs864Zo3Bte8cUxk3VP5gaokaXX5G6qS1JBxl6SGWsR9o3zVQZIHkpxM8uzI2IVJDiR5cbi8YDXnOGlJrkjyzSSHkxxKcucw3nbdSX4wyZNJ/mVY8+8N423XfFqSTUn+OcnXh9ut15zklSTPJHk6ycwwNpE1r/u4b7CvOngQ2DFn7G7gYFVtBQ4Otzt5C7irqj4CXAvcPvz9dl73m8Anq+qjwDZgR5Jr6b3m0+4EDo/c3ghr/kRVbRv5bPtE1rzu484G+qqDqnoCeH3O8E3AvuH6PuDmszmnaauq41X1T8P1N5j9h38Zjddds/53uHne8KdovGaAJJcDvwj8+chw6zUvYCJr7hD3y4DXRm4fHcY2ikuq6jjMhhC4eJXnMzVJtgAfA75D83UP2xNPAyeBA1XVfs3AHwG/DbwzMtZ9zQV8I8lTw1eywITWPK2vHzibFv2qA61/ST4IPAJ8rqq+n8z3195HVb0NbEvyw8CjSX56lac0VUk+A5ysqqeSXL/K0zmbrquqY0kuBg4keX5ST9zhzH2jf9XBiSSXAgyXJ1d5PhOX5Dxmw/6VqvraMNx+3QBV9T/At5j9WUvnNV8H/FKSV5jdWv1kkr+g95qpqmPD5UngUWa3mSey5g5x3+hfdfA4sGu4vgt4bBXnMnGZPUW/HzhcVV8euavtupNsHs7YSfJDwM8Bz9N4zVV1T1VdXlVbmP03/PdV9es0XnOSDyT50OnrwKeAZ5nQmlv8hmqSTzO7X3f6qw72rO6MpiPJQ8D1zH4l6AngXuCvgf3AjwGvArdU1dwfuq5bST4O/APwDO/uxX6B2X33lutO8jPM/iBtE7MnYPur6veT/AhN1zxq2Jb5fFV9pvOak3yY2bN1mN0i/2pV7ZnUmlvEXZL0Xh22ZSRJcxh3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ19P8F5dB6UlXNXQAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Next we plot the number of attemptes on each problem (x) / assignment (color)\n",
+ "from matplotlib.cm import viridis\n",
+ "\n",
+ "assignment_ids = list(students_per_problem.keys().map(lambda x: x[0]))\n",
+ "assignment_ids = [sorted(assignment_ids).index(x) for x in assignment_ids]\n",
+ "colors = [viridis((float(i)-min(assignment_ids))/(max(assignment_ids)-min(assignment_ids))) for i in assignment_ids]\n",
+ "\n",
+ "# There's a slight drop-off by assignment, but overall they're well-attempted\n",
+ "plt.bar(range(0, len(students_per_problem)), students_per_problem, color=colors)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Creating the Class Label: Identifying Struggling Students\n",
+ "\n",
+ "In any student modeling task, our goal is to predict if a student will struggle on the next problem. For this dataset, it's not obvious how to define that struggle.\n",
+ "\n",
+ "We will define struggle as either:\n",
+ "1. Never getting a problem correct or \n",
+ "2. Taking more attempts at a problem than 75% of students before getting it correct.\n",
+ "\n",
+ "The code below justifies this decision by showing that most students get the problem correct _eventually_, and most student with more than the 75th percentil of attempts end up with many more attempts than their peers, indicating struggle."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Aggregate data by problem\n",
+ "\n",
+ "We first get all scored submissions (`Run.Program` events) and aggregate them by SubjectID and ProblemID, counting the number of attempts until a correct submission."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "runs = main_table_filtered[main_table_filtered[PS2.EventType] == 'Run.Program'].copy()\n",
+ "runs['TimeInt'] = pd.to_datetime(runs[PS2.ServerTimestamp]).apply(lambda x: x.value)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "count 10843.000000\n",
+ "mean 4.178179\n",
+ "std 5.739292\n",
+ "min 1.000000\n",
+ "25% 1.000000\n",
+ "50% 2.000000\n",
+ "75% 5.000000\n",
+ "max 93.000000\n",
+ "dtype: float64"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "def get_attempts(rows):\n",
+ " scores = rows[PS2.Score]\n",
+ " # If they scored 1, we return the first time they did so\n",
+ " if (scores.max() == 1):\n",
+ " # Argmax returns the first index of the highest score\n",
+ " # Since the array is 0-indexed, we return +1\n",
+ " return rows[PS2.Score].argmax() + 1\n",
+ " return len(rows.index)\n",
+ " \n",
+ "\n",
+ "scores = runs.groupby([PS2.SubjectID, PS2.AssignmentID, PS2.ProblemID]).apply(get_attempts)\n",
+ "scores.describe()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "count 10843.000000\n",
+ "mean 4.318454\n",
+ "std 5.870877\n",
+ "min 1.000000\n",
+ "25% 1.000000\n",
+ "50% 2.000000\n",
+ "75% 5.000000\n",
+ "max 93.000000\n",
+ "dtype: float64"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Confirm that this is different than just the count of runs\n",
+ "runs.groupby([PS2.SubjectID, PS2.AssignmentID, PS2.ProblemID]).apply(lambda x: len(x.index)).describe()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(10843, 4)\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Attempts</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>3</td>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>5</td>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>12</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>13</td>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Attempts\n",
+ "0 04c32d4d95425f73b3a1d6502aed4d48 439.0 1 1\n",
+ "1 04c32d4d95425f73b3a1d6502aed4d48 439.0 3 2\n",
+ "2 04c32d4d95425f73b3a1d6502aed4d48 439.0 5 3\n",
+ "3 04c32d4d95425f73b3a1d6502aed4d48 439.0 12 1\n",
+ "4 04c32d4d95425f73b3a1d6502aed4d48 439.0 13 2"
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "student_attempts = scores.to_frame('Attempts').reset_index()\n",
+ "print(student_attempts.shape)\n",
+ "student_attempts.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Checking Eventual Success\n",
+ "\n",
+ "Most student get each problem correct eventually, suggesting that the number of attempts is a more meaningful indicator of succeess."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'runs' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)",
+ "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_8216/4164252205.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mcorrect_eventually\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mruns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mPS2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSubjectID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mPS2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mAssignmentID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mPS2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mProblemID\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mPS2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mScore\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
+ "\u001b[1;31mNameError\u001b[0m: name 'runs' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "correct_eventually = runs.groupby([PS2.SubjectID, PS2.AssignmentID, PS2.ProblemID])[PS2.Score].apply(lambda x: max(x) == 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0.9288019920686157"
+ ]
+ },
+ "execution_count": 24,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.mean(correct_eventually)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Attempts</th>\n",
+ " <th>CorrectEventually</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>3</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>5</td>\n",
+ " <td>3</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>12</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>13</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>10838</th>\n",
+ " <td>ffb72475a81de0e95b910ffad039f5c2</td>\n",
+ " <td>502.0</td>\n",
+ " <td>64</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>10839</th>\n",
+ " <td>ffb72475a81de0e95b910ffad039f5c2</td>\n",
+ " <td>502.0</td>\n",
+ " <td>70</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>10840</th>\n",
+ " <td>ffb72475a81de0e95b910ffad039f5c2</td>\n",
+ " <td>502.0</td>\n",
+ " <td>71</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>10841</th>\n",
+ " <td>ffb72475a81de0e95b910ffad039f5c2</td>\n",
+ " <td>502.0</td>\n",
+ " <td>112</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>10842</th>\n",
+ " <td>ffb72475a81de0e95b910ffad039f5c2</td>\n",
+ " <td>502.0</td>\n",
+ " <td>118</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>10843 rows × 5 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Attempts \\\n",
+ "0 04c32d4d95425f73b3a1d6502aed4d48 439.0 1 1 \n",
+ "1 04c32d4d95425f73b3a1d6502aed4d48 439.0 3 2 \n",
+ "2 04c32d4d95425f73b3a1d6502aed4d48 439.0 5 3 \n",
+ "3 04c32d4d95425f73b3a1d6502aed4d48 439.0 12 1 \n",
+ "4 04c32d4d95425f73b3a1d6502aed4d48 439.0 13 2 \n",
+ "... ... ... ... ... \n",
+ "10838 ffb72475a81de0e95b910ffad039f5c2 502.0 64 2 \n",
+ "10839 ffb72475a81de0e95b910ffad039f5c2 502.0 70 2 \n",
+ "10840 ffb72475a81de0e95b910ffad039f5c2 502.0 71 1 \n",
+ "10841 ffb72475a81de0e95b910ffad039f5c2 502.0 112 1 \n",
+ "10842 ffb72475a81de0e95b910ffad039f5c2 502.0 118 1 \n",
+ "\n",
+ " CorrectEventually \n",
+ "0 True \n",
+ "1 True \n",
+ "2 True \n",
+ "3 True \n",
+ "4 True \n",
+ "... ... \n",
+ "10838 True \n",
+ "10839 True \n",
+ "10840 True \n",
+ "10841 True \n",
+ "10842 True \n",
+ "\n",
+ "[10843 rows x 5 columns]"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "student_scores = student_attempts.merge(correct_eventually.to_frame('CorrectEventually'), on=[PS2.SubjectID, PS2.AssignmentID, PS2.ProblemID])\n",
+ "student_scores"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "True 10071\n",
+ "False 772\n",
+ "Name: CorrectEventually, dtype: int64"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "student_scores.CorrectEventually.value_counts()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Choosing a cutoff for \"struggling\"\n",
+ "We choose the 75th percentile of attempts as the cutoff for struggling, and visualize this to verify that it meaningfully separates the \"tail\" of more struggling students from the main body. The chart below shows this for all 50 problems, and suggests that this is a reasonable (though by no means objectively correct) cutoff."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "ProblemID\n",
+ "1 3.0\n",
+ "3 4.0\n",
+ "5 4.0\n",
+ "12 2.0\n",
+ "13 11.0\n",
+ "dtype: float64"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "problem_attempt_75th = student_scores.groupby(PS2.ProblemID).apply(lambda x: x.Attempts.quantile(0.75))\n",
+ "problem_attempt_75th.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAA20AAAI/CAYAAADkwzGCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABz+UlEQVR4nO39f4xc5X33/7/emNKqiatg7N2uvWwWok1UAsRNVkQokUVlLcltEA5/BNhIH2zZwgjFSotA6eaToNv+VOS79L4JqkRaYW6jOFHrAGodWwQcnJVX0QfRgEGLsUNdQ9g2W7b+EYNC/SkEw/v7x8xsxuuZ2flxnXOuc+b5kFa7c2Z25prXnHPmXOe6znWZuwsAAAAAEKfzsi4AAAAAAKA+Km0AAAAAEDEqbQAAAAAQMSptAAAAABAxKm0AAAAAEDEqbQAAAAAQsfMXeoCZXSzp+5L+WNIHkra5+9+Y2RJJj0oalDQt6SZ3f7P8P9+QtFHS+5K+5u4/afQaS5cu9cHBwfbfRYG98MILJ919Wav/R6b1tZupRK71kGky2P7DI9PwyDQ8Mg2PTMMj0/AaZbpgpU3SGUl3ufuLZrZY0gtmtk/SekkT7j5uZmOSxiT9pZldJukWSZ+UtFzST83s4+7+fr0XGBwc1IEDB1p7V13CzP6tnf8j0/razVQi13rINBls/+GRaXhkGh6Zhkem4ZFpeI0yXbB7pLvPuvuL5b/flvSKpBWS1kraUX7YDklfKv+9VtIP3f1dd39d0quSrmq79AAAAADQxVq6ps3MBiX9qaSfS+p191mpVLGT1FN+2ApJv6r6t5nyMgAAAABAi5qutJnZhyX9o6S/cPffNHpojWVe4/k2mdkBMztw4sSJZosBAAAAAF2lmWvaZGa/p1KF7e/d/Z/Ki4+ZWZ+7z5pZn6Tj5eUzki6u+vd+SW/Mf0533yZpmyQNDw+fVakbHPtxzXJMj1/XTHFRA5mGR6bhkWl4ZBpevUwlcm0XmYZHpuGRaTL4nmrOgi1tZmaStkt6xd2/U3XXHknryn+vk7S7avktZvb7ZnaJpCFJz4UrMgAAAAB0j2Za2j4n6f+S9LKZTZWX/d+SxiU9ZmYbJf27pC9LkrsfNrPHJP1CpZEnv9po5EgAAAAAQH3NjB75/7q7ufuV7r6y/POku//a3Ve7+1D596mq/7nX3T/m7p9w96eSfQtAyYYNG9TT06PLL798btmWLVu0YsUKrVy5UitXrtSTTz45d5+ZfcPMXjWzI2b2hSzKDADoDnxHAehES6NHAjFbv3699u7de87yO++8U1NTU5qamtKaNWskSfPmE/yipL81s0VplhcA0D1a+Y6S9AfiOwpAFSptKIxVq1ZpyZIlzT6c+QQBAKlp8TvqI+I7akG0XqKbUGlD4T344IO68sortWHDBr355puVxcwnCBQEB27IszrfUReI76gF0XqJbkKlLQMcYKTnjjvu0GuvvaapqSn19fXprrvuqtzV1HyCEnMKArHjwA151eA7qha+o+ah9RLdhEpbBjjASE9vb68WLVqk8847T7fddpuee25u9omm5hOUSnMKuvuwuw8vW7Ys4RIDaBUHbsirBt9RvxXfUW3rtPWymyvCiBeVtgxwgJGe2dnZub937dpV3brJfIJAwXHghtg1+I56S3xHtSVE6yUVYcSISltE6NfemdHRUV199dU6cuSI+vv7tX37dn3961/XFVdcoSuvvFL79+/XAw88IKk0n6CkynyCe8V8gkChcOCG2LTyHSXpHfEd1ZYQrZdAjJqZXBspuOOOO3TPPffIzHTPPfforrvu0iOPPFLv4XX7tUvaJEkDAwMJlTReO3fuPGfZxo0b6z7e3e+VdG+CRQKQkd7e3rm/b7vtNl1//fWVmxy4IRN8R6VjdnZWfX19kuq2Xn5H0nLReomcodIWiRAHGO6+TdI2SRoeHq5ZsQOAbsCBG1B8o6Ojmpyc1MmTJ9Xf36+tW7dqcnJSU1NTMjMNDg7qoYceqjz8HUm7VGq9PCNaL5EzVNoiwQEG0B02bNigJ554Qj09PTp06JAk6dSpU7r55ps1PT2twcFBPfbYY7rwwgsllUaPlbRR0vuSvubuP8ms8JHiwC28Wuvpli1b9PDDD6vSVfTb3/723KBZrKfIAq2X6CZU2jLAAQbQvdavX6/Nmzfr1ltvnVs2Pj6u1atXa2xsTOPj4xofH9d9990nnT167HJJPzWzj7MPOBsHbuHVWk+l0ijHd9999/yHs54CQMKotGWAAwzkRbNn2ys4276wVatWaXp6+qxlu3fv1uTkpCRp3bp1uuaaayqVto9I+q67vyvpdTOrjB77bIpFRheqtZ428BGxngJAohg9EkBdrcwpaGaXiTkF23Ls2LG57tF9fX06fvx45S5Gj0VUGOUYALJBpQ1AXS3OKbhWzCmYhrqjxzKnGJIUYhoF1lMgbhs2bFBPT0/12Ao6deqURkZGNDQ0pJGRkeoTNjKzb5jZq2Z2xMy+kEWZuwWVNgAtq3O2fYWYtLgtvb29c5Pszs7Oqqenp3JXS6PHMqcYkhRi/ivWUyButXrYVK67Pnr0qFavXq3x8fHKXdXXs9LDJmFU2gC0pMHZdqvxcCYtbsINN9ygHTt2SJJ27NihtWvXVu56S6XRY3/fzC4Ro8ciQ5UTC1LdUY5ZT4Gcq9XDZvfu3Vq3bp2k0nXXP/rRjyp3fUT0sEkNA5EAaEmDOQVnxKTFC6o1euzY2Jhuuukmbd++XQMDA3r88ccrD2f0WGSCUY4BVHDddRyotAFoSa05BQ8fPixJeyT9A3MKNlZr9FhJmpiYqLmc0WORBUY5BtCmutezStokSQMDA6kWqCiotAGoq9mz7Y8++qjc/bCZPSbOtgMAUBiV6677+vo6uu5a0jZJGh4erlmxQ2NU2gDUxdl2AAC6W+W667GxsXrXXdPDJgVU2gAAAABw3XXEqLQBAAAA4LrriDHkPwAAAABEjEobAAAAAESMShsAAAAARIxKGwAAAABEjEobAAAAAESMShsKY8OGDerp6dHll18+t+zUqVMaGRnR0NCQRkZG9Oabb87dZ2bfMLNXzeyImX0hizIDALoD31EAOkGlDYWxfv167d2796xl4+PjWr16tY4eParVq1drfHxckmRml0m6RdInJX1R0t+a2aK0ywygcxwMIw9a+Y6S9AfiO2pBbPvoJlTaMsBOJhmrVq3SkiVLzlq2e/durVu3TpK0bt06/ehHP6rctVbSD939XXd/XdKrkq5Kr7QAQuFgGHnQ4nfUR8R31ILY9tFNqLRlgJ1Meo4dO6a+vj5JUl9fn44fP165a4WkX1U9dKa87BxmtsnMDpjZgRMnTiRZXABt4GAYedXgO+oC8R21ILZ9dBMqbRlgJxMFq7HMaz3Q3be5+7C7Dy9btizhYgEIIcTBMBAZvqOaQEUYRUWlLRIcYCSjt7dXs7OzkqTZ2Vn19PRU7pqRdHHVQ/slvZFu6QBEoubBMAduSFqD76jfiu+oNFARRm5QacsnDjCadMMNN2jHjh2SpB07dmjt2rWVu/ZIusXMft/MLpE0JOm5bEoJILQQB8PdfODGtdfpaPAd9Zb4jmoLFWEUFZW2SHCA0bnR0VFdffXVOnLkiPr7+7V9+3aNjY1p3759Ghoa0r59+zQ2NiZJcvfDkh6T9AtJeyV91d3fz7D4UeLADXnFwXBnuPY6vFa+oyS9I76j2sK2j6I6P+sCoKSykxkbG6u3k/mOpOViJ1PXzp07ay6fmJioudzd75V0b4JFyr3169dr8+bNuvXWW+eWVQ7cxsbGND4+Xm8aheWSfmpmH+dAA0kbHR3V5OSkTp48qf7+fm3dulVjY2O66aabtH37dg0MDOjxxx+vPPwdSbtUOhg+Iw6Ga1q1apWmp6fPWrZ7925NTk5KKl17fc011+i+++6TStdef9fd35X0uplVrr1+NsUiR4/vqPDY9tFNFqy0mdkjkq6XdNzdLy8vWyLpUUmDkqYl3eTub5bv+4akjZLel/Q1d/9JIiXPMXYyyItmD9zK5qZREAduSBEHw+ng2mvEhm0f3aSZlrbvSXpQ0verlo1JmnD3cTMbK9/+S860N4edDPJsgWkU/rnqoRy4Ad2r7rXXkjZJ0sDAQKoFAoA8W/CaNnf/maRT8xavlbSj/PcOSV+qWs7w9EB3anoaBQbNAfKHa68BIDvtDkTS6+6zklT+XdlzNz1hMYB8CjGNAgduQP4wwAMAZCf06JGcaQcKjmkUgOJjpEMAiEu7o0ceM7M+d581sz5JlYtaWjrTLmmbJA0PD9es2AHIVrOD5vyv//W/5O6Hzaxy4MagOUCOce01AMSl3UrbHknrJI2Xf++uWv4PDE8PFAMHbgAAANlrZsj/nZKukbTUzGYk/U+VKmuPmdlGSf8u6cuSONMOAAAAAIEtWGlz99E6d62u83jOtAMAAABAIO12jwQABDY4OKjFixdr0aJFOv/80u7ZzJZIelTSoKRpSTe5+5uZFRIAAKQu9OiRAIAO7N+/X1NTUzpw4EBl0ZikCXcfkjRRvg0AALoILW0AELe1Kl1XLEk7JE1K+susCgMA6E70BskWlTYAiISZ6dprr5WZ6fbbb68s7nX3WUkqT7PSU/8ZAABIzv79+7V06VJJpe8s/a43yLiZjZVvc2IxAVTaACASzzzzjJYvX67jx49rZGREkj7c7P+a2SZJmyRpYGAgWJlufuhZSdKjt18d7DkBAIVBb5CUcE0bAERi+fLlkqSenh7deOONkvQhScfMrE+Syr+P1/pfd9/m7sPuPrxs2bKUSgwA6BaV3iCf+cxntG3btsris3qDSKI3SEJoaQOACJw+fVoffPCBFi9erNOnT+vpp5+WpP+WtEfSOpXmx1wnaXeGxQQAdKlYeoN0aw8QKm0AEIFjx45VWtd05swZfeUrX9Gzzz77G5Uqa4+Z2UZJ/y7pyxkWEwDQpeb3Bjl48OBcb5DyNdcNe4NI2iZJw8PD3szrDY79uO59n71kSYulzz8qbQAQgUsvvVQvvfTSWcu+9a1vyd1/LWl1NqUCAIDeIDGg0oauUD1MraQ/kRimFgAQB4ZSD49Mw6I3SPaotEWICkYyKsPUmtkr5UUMUwsUHAduyAuGUg+PTMOhN0j2GD0yUvv379fU1JQkza9gDEmaKN9GZ9aqNDytyr+/lF1RACSlsj89cOBAZRH7U+QB31HhkSlyi5a2/GAejA5UT1osaWl5MZMWA92J/WkHaL0Mr/o76vbbb68s5juqA2SKoqHSFiEqGOFVD1Pb29vbY2armv3fdoap7YbhaOnGm2+NRuWaHr8uxZIkq5MDN7b9+uh2FlbWQ6kXcb3NMtMi5ons0T0yQs8884xefPFFPfXUU5LUdAXDzDaZ2QEzO3DixImmX+/mh56d28EUVfUwtZLeknSVmLS4Y3TjReyq96ff/e53pRYO3Nj2W0K3sw7MH0pd0txQ6hLfUe0gUxQNlbYItVvBYCdT2+nTp/X222/P/S3pjyQd0u+GqZUYpjYUDtwQlU4O3FBbpfXyM5/5jLZt21ZZfFbrpSR6gzRp/ndUjaHUJb6jWkKmKCK6R0Zm/jwYOreCwTwYLZo/TK2kt9x9r5k9L4apbVsn3Xhb7XpCVxO0g3mFkpF2t7Oib/8MpR4emaKIclVpK/qOW6KCkYT5w9Sa2X9KYpjaDnVynaC7b5O0TZKGh4c9sUKiq3Hgloz5rZcHDx6ca70sn6xp2O1MbPtnYSj18MgURZSrSls3oIKBvGjUjXehAzcgDRy4hUfrJQBkg2vaALSM6wSB7nTs2DF9/vOf16c+9SldddVVuu666ySp0no5YmZHJY2UbwMAAqGlDUDL6MYLdCdaLwEgG1TaALSMbrwAAADpodLW5aon2J0/2W6RJtgF0LxaE28Pjv2YfQIAABmh0gYERkUYAAAAITEQCQAAAABEjJY2AAAC+fnrpyTRyo641eoCDSBuVNoAAE1pdKBHpQRAN+NaYCSN7pEAAAAAEDFa2gBEr16XM4kWHgAAus3PXz/VdccEVNoAAACABNCtHKHQPRIAAAAAIparljZG5QKAONU7m8z+GZ3iuz89tAoB8cpVpQ3IOw5sAQAA0lWEExJU2gAAQO4U4SAsTzjpCGSLShvqYgedHg4+gGJjfwoAySvyxPFU2gAAyAgnbAAAzUhs9Egz+6KZHTGzV81sLKnX6SZkGh6Zhkem4ZFpeGQaHpmGR6bJINfwyDR5ibS0mdkiSd+VNCJpRtLzZrbH3X+RxOt1g5gyLUo3n5gyLQoyDS/vmabVktTK6+Ql03a6+WS1H85LpnlCpsmIKVeOp9CKpLpHXiXpVXf/pSSZ2Q8lrZWUyIfXJd1LUs20S+Q600h39qlnGmkOIeV6PW0kw0pJYTPNUFSZFmS/EFWmBRJ9ru0c12Z8LBx9po3kZX+RVKVthaRfVd2ekfTZhF6roXY+iLQ+vBZfJ5pM64nhrHDRMpXSuag24GcXTaahc0ujVSj2TGMQ6HMtbKZpXYRfY13NRaYhD4bb2Se0+Pq5zzSkgJWVXOSahiJ+94eU9bo9X1KVNquxzM96gNkmSZvKN//LzH4t6WRC5TmH3Rf8f5YqUPnnvc5HK4trPHShTI+ELFcSyu818TK2m6mU/braqnnvNbFsA2d6JECRQr7Xc56rnX1Gveeqp8ZrfFSdZRrj9p96mWqsq0XLNE1LJZ0MmGk0+9NW950d7BOaeb5CZBpSVT7t7lM7OZ6KJtPAx68t788CZfqumR1q5XULaC77OpmeI6lK24yki6tu90t6o/oB7r5N0rbKbTM74O7DCZUncSmUv+VMUypXxzIs44KZSvleVzMoa1uZhhDyvcb0XGZ2tdrMNMZ1NYYyFS3TNNV7/+1mGmueMZSraJmGEuD9dd0xaj0B31dLmRY1z1a0k0FSo0c+L2nIzC4xswsk3SJpT0Kv1S3INDwyDY9MwyPT8Mg0PDINj0yTQa7hkWkKEmlpc/czZrZZ0k8kLZL0iLsfTuK1ugWZhkem4ZFpeGQaHpmGR6bhkWkyyDU8Mk1HYpNru/uTkp5s4V+CdpXKQOLlbyNTKR+5ZlbGAmdakXpZ28w0hJDvNarn6iDTGNfVKMpUsEzTVPf9F2x/GkW5CpZpKFnsU4uaabD31WKmRc2zFS1nYO7nXNMKAAAAAIhEUte0AQAAAAACiKLSZmZfNLMjZvaqmY1lXZ5GzOxiM9tvZq+Y2WEz+/Py8iVmts/MjpZ/X5hxOaPMNC/51RJrplK+cw3BzKbN7GUzmzKzAy3+7yNmdrx6+OF2c6vzXFvM7D/KZZsyszWtlK8dMayrDdbJ1PMIIYZMs1Br2wq1X4kl06LsP2PJM5QYPpeiZBpDllVlKUSmzQqavbtn+qPSBYuvSbpU0gWSXpJ0WdblalDePkmfLv+9WNK/SrpM0l9LGisvH5N0H5nmM7+8ZZrnXAO+/2lJS9v831WSPi3pUNWytnKr81xbJN2dYhZRrKsN1slU8yhSphm993O2rRD7lZgyLcL+M6Y8i/K5FCnTrLMsYqZZZB9DS9tVkl5191+6+28l/VDS2ozLVJe7z7r7i+W/35b0ikozwa+VtKP8sB2SvpRJAUuizTQn+dUSbaZSrnPNnLv/TNKpeYvbyq3Oc6UtinW1wTqZR1FkGpEQ+5VoMi3I/jOaPEOJ4HMpTKYRZFlRmEybFTL7GCptKyT9qur2jHLyxW5mg5L+VNLPJfW6+6xU+oAk9WRYtFxkGnF+teQiUyl3uYbikp42sxfMbFOA5wud22YzO1juPpl095Po1tV566SUbh4hRJdpimptWyG2jygzzfH+M8o8Q8nocylkphmv44XMtFmdZh9Dpc1qLIt+SEsz+7Ckf5T0F+7+m6zLM0/0mUaeXy3RZyrlMtdQPufun5b0PyR91cxWZV2gKn8n6WOSVkqalXR/wq8X1bpaY51MO48Qoso0ZUltW9FlmvP9Z3R5hpLh51K4TCNYxwuXabNCZB9DpW1G0sVVt/slvZFRWZpiZr+nUvB/7+7/VF58zMz6yvf3STqeVfkUeaY5yK+WqDOVcptrEO7+Rvn3cUm7VOqC0Ylgubn7MXd/390/kPRwgLItJJp1tdY6mUEeIUSTadrqbFshto+oMi3A/jOqPEPJ+HMpVKaRrOOFyrRZobKPodL2vKQhM7vEzC6QdIukPRmXqS4zM0nbJb3i7t+pumuPpHXlv9dJ2p122apEm2lO8qsl2kylXOfaMTP7kJktrvwt6VpJhxr/14KC5VbZKZfdqM7LtpAo1tV662QGeYQQRaZpa7Bthdg+osm0IPvPaPIMJYLPpTCZRpBlRWEybVbQ7EOPktLOj6Q1Ko2m8pqkb2ZdngXK+nmVmnIPSpoq/6yRdJGkCUlHy7+XkGl+88tTpnnPNcB7v1SlEaheknS41c9G0k6Vuum9p9JZwI3t5lbnuX4g6eXyZ7NHUl8KmWS+rjZYJ1PPoyiZZvCea25bofYrsWRalP1nLHkW6XMpSqYxZFm0TLPI3spPCAAAAACIUAzdIwEAQESsxYnizewb5clyj5jZF7IpNQAUFy1tAADgLOVRIv9L0vfd/fLysi2S/svd//e8x16mUtfgqyQtl/RTSR939/dTLTQAFBgtbQAA4Cze2kTxayX90N3fdffXJb2qfIwKCgC5cX7WBZCkpUuX+uDgYNbFiNILL7xw0t2Xtfp/ZFpfu5lK5FoPmSaD7T88Mm3e5ZdfrldffVXDw8MuSX19ffr1r3+tP/zDP/xff/iHf6j+/n6df/75kvSOpNuq/rWpCXO7MdNmsZ6GR6bhkWl4jTKNotI2ODioAwcOZF2MKJnZv7Xzf2RaX7uZSuRaD5kmg+0/PDJt3vT0tK6//vq5933s2DEtXbpUZqZ77rlHs7OzeuSRR2Rmb9f495rXXpjZJkmbJGlgYKDrMm0W62l4ZBoemYbXKFO6RwIAgAX19vZq0aJFOu+883Tbbbfpueeeq9z1WzU5Ya67b3P3YXcfXrasrcZ5AOhKVNoysGHDBvX09Ojyyy+fW3bq1CmNjIxoaGhIIyMjevPNN+fuY1QuAEDWZmdn5/7etWtX9XfYW5JuMbPfN7NLJA1Jeu6cJwAAtC2K7pHzDY79uOby6fHrUi5JMtavX6/Nmzfr1ltvnVs2Pj6u1atXa2xsTOPj4xofH9d9990nSX+g0ozxn1R5VC4za3lUrqJnmgUyDY9MwyPT8OplKhUn19HRUU1OTurkyZPq7+/X1q1bNTk5qampKZmZBgcH9dBDD1Ue/o6kXZJ+IemMpK+G+o6SipNp2sg0PDJNBt9TzYmy0lZ0q1at0vT09FnLdu/ercnJSUnSunXrdM0111QqbR+R9F13f1fS62ZWGZXr2RSLjC61YcMGPfHEE+rp6dGhQ6XpmrZs2aKHH35Yla5N3/72t+ceb2bfkLRR0vuSvubuP0m/1AA6tXPnznOWbdy4se7j3f1eSfcmWCQA6Gp0j4zEsWPH1NfXJ6k0Qtfx48crd10g6VdVD21qVC4ghPXr12vv3r3nLL/zzjs1NTWlqakprVlTml+3PFdTpVX4i5L+1swWpVleAACAIqLSlk91R+UyswNmduDEiRNplylzta4V3LJli1asWKGVK1dq5cqVevLJJ+fu41rBha1atUpLlixp9uHM1QQAAJAAKm2R6O3tnbvIe3Z2Vj09PZW7GJWrSbQKpefBBx/UlVdeqQ0bNlQPmrNCtAoDAAAER6UtEjfccIN27NghSdqxY4fWrl1buestMSpXU2gVSscdd9yh1157TVNTU+rr69Ndd91VuctqPJxWYQAAgA5RacvA6Oiorr76ah05ckT9/f3avn27xsbGtG/fPg0NDWnfvn0aGxurPPwdSY+pNCrXXrUxKle3o1UorAZzNc2IVmEAQEq4LALdhEpbBnbu3KnZ2Vm99957mpmZ0caNG3XRRRdpYmJCR48e1cTExFktRu5+r7t/zN0/4e5PZVj03KFVKLwGczXtEa3CAICUtHJZhM6eQonLIpA7VNpQaLQKdaZWq/DXv/51XXHFFbryyiu1f/9+PfDAA5Ikdz8sWoUBAClp8bKIj4jLIpBjVNpQaLQKdaZWq/APfvADvfzyyzp48KD27NkzN1WFRKswACB7dS6LaHoKpW7uYVOry+mpU6c0MjKioaEhjYyMVGdKl9MUUWlDYdAqBABAd2twWUQtNS+L6OYeNrW6nI6Pj2v16tU6evSoVq9erfHx8cpddDlN0flZFwAIZefOnecs27hxY93Hu/u9ku5NsEgAACBFvb29c3/fdtttuv766ys3m55CqZutWrVK09PTZy3bvXu3JicnJUnr1q3TNddco/vuu08qdTn9rru/K+l1M6t0OX02xSJ3DVraAAAAEsZIh+locFnEW+KyiLYcO3Zs7lKIvr4+HT9+vHIXXU5TREsbAABAwtavX6/Nmzfr1ltvPWv5nXfeqbvvvnv+w6u7nS2X9FMz+zjd+M82OjqqyclJnTx5Uv39/dq6dasmJyc1NTUlM9Pg4KAeeuihysPfkbRLpcsizojLIpJSt8uppG2SNDw8XPMxaGzBSpuZPSLpeknH3f3y8rItkm6TVKkq/9/u/mT5vm9I2ijpfUlfc/efJFBuAACA3KjV7ayBj4huZwvisoh09Pb2anZ2Vn19fZqdnVVPT0/lLrqcpqiZ7pHfU+niwvkecPeV5Z9Khe0ycUEiANTFyFwAqjHSIWJ3ww03aMeOHZKkHTt2aO3atZW73hJdTlOzYKXN3X8m6VSTz7dWzIEBAHUxMheACkY6RGxqjcQ9Njamffv2aWhoSPv27dPY2Fjl4e+IkbhT08k1bZvN7FZJByTd5e5vqnQW6J+rHlP3zBAAdCNG5gJQwUiHiE2tLqeSNDExUXM5XU7T0+7okX8n6WOSVkqalXR/ebnVeGzNM0M05wNASYiRuQDkDyMdAmhWW5U2dz/m7u+7+weSHtbvukDOqMkzQzTnA0BbOBE2D9cJIg9qdTv7+te/riuuuEJXXnml9u/frwceeKDycLqdAThLW5U2M+urunmjpEPlv/eIM0MA0JLKyFyS2h6Zq5tPhHGdIPJg586dmp2d1XvvvaeZmRlt3LhRP/jBD/Tyyy/r4MGD2rNnz1yLu1TqdubuH3P3T7j7UxkWHUAEFqy0mdlOla6f+ISZzZjZRkl/bWYvm9lBSX8m6U5JcvfD4swQALSEkbk6s2rVKi1ZsuSsZbt379a6deskla4T/NGPflS56yNiwCwAQM4sOBCJu4/WWLy9weO5IBEA6qg1GezY2Jhuuukmbd++XQMDA3r88ccrD2cy2DaFuE7QzDZJ2iRJAwMDiZUVAICFdDJ6JACgRYzMFaW6Q6lL2iZJw8PDNR8DAEAa2h09EgCAaIW4TrCbMbgLAMSFShsAoHC4TrAzDO4CAHGh0gYAyLVaQ6mPjY1p3759Ghoa0r59+zQ2NlZ5OEOpN4HBXQAgLlzTBgDINa4TTAeTwANAdmhpAwAAoTEJPAAERKUNQF0MRgCggkngASA7VNoA1NXKYARmdpkYjAAoLAZ3AYDsUGlDYdAqFF6LgxGsFYMRAIXA4C4AEBcqbSgMWoXS0WAwghViMAKgEHbu3KnZ2Vm99957mpmZ0caNG3XRRRdpYmJCR48e1cTExFkndNz9Xnf/mLt/wt2fyrDo6CKcrEU3odKGwqBVKHNWYxmDEQAAEsF8gugmVNpQaCFahahgnK3BYAQzYjACAEBKmE8Q3YRKG7pV061CVDDO1mAwgj1iMAIAQIaYTxBFRaUNhRaiVaibtTIYgbsfFoMRAADygy78yA0qbSg0WoU6w2AEABAGg2akg/kEUVQLVtrM7BEzO25mh6qWLTGzfWZ2tPz7wqr72MkgE7QKAQBixaAZ6WA+QRTV+U085nuSHpT0/aplY5Im3H3czMbKt/9y3jDqyyX91Mw+zsEw0rBz586ayycmJmoud/d7Jd2bYJEAAJBUGjRjenr6rGW7d+/W5OSkpNKgGddcc43uu+8+qTRoxnfd/V1Jr5tZZdCMZ1MscvRGR0c1OTmpkydPqr+/X1u3btXY2Jhuuukmbd++XQMDA3r88ccrD39H0i6VTtaeESdrkTMLVtrc/WdmNjhv8VpJ15T/3iFpUtJfqmoYdbGTAQAAqCvEoBlmtknSJkkaGBhIrKwx4mQtukm717T1uvusJJV/VzoMM7kuAABAeIxwDHSx0AORMLkuAABAE0IMmgGgO7RbaTtmZn2SVP5dac9nct0ABgcHdcUVV2jlypWS9CdS48FfAABA/jBoBoBmtVtp2yNpXfnvdZJ2Vy1nJxPA/v37NTU1JUmvlBdVBn8ZkjRRvg0AAHKglRGOVRo0gxGOAcxZcCASM9up0qAjS81sRtL/lDQu6TEz2yjp3yV9WSoNo25mlZ0MI/OEVW/wFwAAEDkGzQDQiWZGjxytc9fqOo9nJ9MhM9O1114rM5OkpeXFZw3+YmY9Nf4v9RGkbn6oNDDoo7dfncrrAQAAAN2mmXnakLJnnnlGy5cv1/Hjx9Xb29tjZqua+T933yZpmyQNDw/XHAAGAAAAaNXg4KAWL16sRYsW6fzzS1UIM1si6VFJg5KmJd3k7m9mVsgCCz16JAJYvny5JFVGkXpLpbnu6g3+AgAAACSuMubCgQMHKosYcyElVNoic/r0ab399ttzf0v6I0mHVH/wFwAAACALa1Uaa0Hl31/KrijFRvfIyBw7dkw33nijJOnMmTOS9Ja77zWz51Vj8BcAxUHXEwBArKrHXLj99tsrixcccwFhUGmLzKWXXqqXXnpp7raZ/ackufuvVWfwFwDFsX//fi1dWhp/qDwYUaXrybiZjZVvM3IsACBV1WMujIyMSNKHm/3fkIPldesgeFTaACBuiUz3MTj247r3TY9f1+nTR4PWSwAIo3rMhRtvvFEHDx78kMpjLpRb2eqOucBgeZ3jmjYAiESl68lnPvMZbdu2rbL4rK4nkuh60iIunAeAzswfc+Hpp5+WpP8WYy6khpY2AIhELF1PukAirZcAUFTzx1z4yle+omefffY3ksbFmAupoNIGAJGg60l4XDgfHl1Oge4zf8wFSfrWt77FmAspotIGABE4ffq0PvjgAy1evLhe15Nx0fWkZVm1Xhb9QnkGzAGAdFFpA4AI0PUkGbRepoYupwCCajRg1mcvWZJiSeJApQ1dobo7j6Q/kejO0ykyDYuuJ+Fl0Xo5/yCjcrtII3LS5RQA0kelDV2j0p3HzF4pL6I7T4fIFDGj9TIZdDlFHnDtJYqGIf/Rzdaq1I1H5d9fyq4ohUGmiEal9fKll17S4cOH9c1vflOS5O6/dvfV7j5U/n0q46Lmyvwup5LmupxK0kJdTt192N2Hly1bllKJ0a2Y7gNFQksbukJ1dx5JS8uL6c7TgU4ybfVsO2fYgTgwYE4yaBVKDddeIrc6qrSZ2bSktyW9L+mMuw+zk0GMqrvz9Pb29pjZqmb/t53uPN1QyegkUwZ4APKJLqfJYUTOsLj2EkUToqXtz9z9ZNVtdjI50mhkniJdOF/dnUfSW5KuEiPIdaSTTAHkEwPmpIpWoQ6kfe3lzQ89W+gTtcheEte0cU0LonL69Gm9/fbbc39L+iNJh/S77jwS3XlaQqYAEE6lVegzn/mMtm3bVll8VquQJFqFWsC1lyiaTlvaXNLTZuaSHiq3SND0jKjM784j6S1332tmz4vuPG0hUwAIh1ahsLj2EkXUaaXtc+7+Rrlits/M/qXZf+xk2F+gFfO785jZf0qiO08HyBQAwmES+LC49hJF1FGlzd3fKP8+bma7xHVChVLverciXesGAIhb0Scsp1UoPK69RBG1fU2bmX3IzBZX/pZ0rbimBQAAoGnHjh3T5z//eX3qU5/SVVddpeuuu06SKq1CI2Z2VNJI+TaALtVJS1uvpF3lYWnPl/QPXNMCAADQPFqFADSj7Uqbu/9S0qdqLGcnAwAAACARP3/9VM3LeIrSbbqWEPO0AahSvROZv0Mp8s4EAAAAyUhinjYAAAAAQCC0tAEAAAAd6MbuekgXlTa0rNZOaXDsx+yYgIJgug8AAOJCpQ0AgIRREQYAdIJr2gAAAAAgYrS0AQAAdBGuv0JRFblXAy1tAAAAABAxWtoQTL2zG1IxznAgOz9//ZSkxuvYfKxzAACgKGhpAwAAAICIUWkDAAAAgIjRPRIAAOQOXfIBdBNa2gAAAAAgYrlqaSvyMJ4AELtWBoKpYP8MAEDnclVpQ35xsAcA56KLHwCgGYlV2szsi5L+RtIiSf/H3ceTeq1uQabhxZJpkQ7cYsm0SMg0PDINL6ZMi9IzJ+1Mi/Rd1EiauZIpQkmk0mZmiyR9V9KIpBlJz5vZHnf/RRKv10iBdtzRZJqWdj67Vv4ni0zbaXHMk25cT5NW5Eyz2j8XOdOskGl4ZJqMmHLlGDUOeekNllRL21WSXnX3X0qSmf1Q0lpJiXx4oQ+EI92IUs00ZgE/bzINL5pM87ITbkI0mcasxbPZZBpe7jNtdZ+Rwv4iqkwjPTZqR1S5tirSzyHXmYaUZMtqUpW2FZJ+VXV7RtJnqx9gZpskbSrf/C8z+7WkkwmV5xx2X9v/s1QpllPSR8u/28n0SPLFk5R+Ji2/5rzPu+lMpURzbeo9tLCuhv4cWnm+TjNNdfuvp0bWWazb1T6q4mU63zkZt7N/bqEMnWR6RCmuE6FzSEAli9yspyH3pwl/Pqyn7av33kMcT6WSa5OZnlWWjD6HomXayvO1XO4my/DRenckVWmzGsv8rBvu2yRtm/sHswPuPpxQeYLJsJwtZ5qWLDIJ9JoLZioll2vo3CJ5vrYyjXX7j6FcZvblGotzm+l8Ge0/2sq0/L+5yDUN1VkUbT2NoVysp+1p4r23fTwVU64xlUUFybQVWZQ7qXnaZiRdXHW7X9IbCb1WtyDT8Mg0PDINj0zDI9PwyDQ8Mk0GuYZHpilIqtL2vKQhM7vEzC6QdIukPQm9Vrcg0/DINDwyDY9MwyPT8Mg0PDJNBrmGR6YpSKR7pLufMbPNkn6i0tCfj7j74QX+LfVufW3KpJxtZpqWLDLp+DUjyDR0bpk/XweZxrr9Z16uAmY6X+rl7HDbz0uuaZjLooDraeblYj1tW8P3XqBcoylLgTJtRfqXI7mf0z0aAAAAABCJpLpHAgAAAAACoNIGAAAAABGLotJmZl80syNm9qqZjWVdHkkys4vNbL+ZvWJmh83sz8vLl5jZPjM7Wv59YdZlzZKZTZvZy2Y2ZWYHEnqNR8zsuJkdqlqW68+h09xCZ1Ln+baY2X+UyzhlZmtaLWcTrxvFtl+07T2WXKs1yDjx9SyEGDNNU619VqfbRyyZsv3nWxLrZo3XyDTTNN5j2rLOtBXR5O/umf6odMHia5IulXSBpJckXRZBufokfbr892JJ/yrpMkl/LWmsvHxM0n1ZlzXjnKYlLU34NVZJ+rSkQ1XLcv05dJpb6EzqPN8WSXcnmEE0236RtveYcm0y40TXsyJnmnIG5+yzOtznRJMp23++f0KvmzFmmvR7zOAzyzzTPOYfQ0vbVZJedfdfuvtvJf1Q0tqMyyR3n3X3F8t/vy3pFZVmfF8raUf5YTskfSmTAnYRd/+ZpFPzFnf15xA6kzrPl7Rotv2Cbe/R5FqtQcZ5EGWmEehk+4gmU7b/Qgr52cWaaR7Xz4pYM21F6vnHUGlbIelXVbdnFNkXuZkNSvpTST+X1Ovus1JpRy+pJ8OixcAlPW1mL5jZphRfN++fQxK5JZHJZjM7WO4+GbrpP8ptvwDbe5S5VpuXsZTsehZC9JmmoNY+q5PtI8pM2f5zKfS6OV8MmSb9HtMWQ6atiCL/ROZpa5HVWBbNPARm9mFJ/yjpL9z9N2a1itvVPufub5hZj6R9ZvYv5VYbNJaH3P5O0l+ptD3+laT7JW0I+PzRbfsF2d6jy7VajYyTXs9CiDrTlJyzz+rw+aLLlO0/t0Kvm/PFkGnS7zFtMWTaiijyj6GlbUbSxVW3+yW9kVFZzmJmv6fSDvzv3f2fyouPmVlf+f4+ScezKl8M3P2N8u/jknap1OSdhlx/DgnlFjQTdz/m7u+7+weSHg5UxmpRbfsF2t6jyrVarYxTWM9CiDbTtNTZZ3WyfUSVKdt/fiWwbs6XeaYpvMe0ZZ5pK2LJP4ZK2/OShszsEjO7QNItkvZkXCZZ6RTbdkmvuPt3qu7aI2ld+e91knanXbZYmNmHzGxx5W9J10o61Pi/gsnt55BgbkEzqeyMym5U+M82mm2/YNt7NLlWq5dxCutZCFFmmpYG+6xOto9oMmX7z6+E1s35Ms00pfeYttysp1Hln/RIJ838SFqj0mhNr0n6ZtblKZfp8yo11R6UNFX+WSPpIkkTko6Wfy/JuqwZZnSpSiP+vCTpcFKfnaSdkmYlvafS2ZmNef4cQuQWOpM6z/cDSS+Xt4E9kvoSyCKKbb9o23ssuTaZceLrWVEzTfG919xndbp9xJIp239+f5JaN2PKNK33mMFnl4v1NKb8rfzCAAAAAIAIxdA9EgAAAABQB5U2AAAAAIhYDEP+a+nSpT44OJh1MaL0wgsvnHT3Za3+H5nW126mErnWQ6bJYPsPj0zDI9PwyDQ8Mg2PTMNrlGkUlbbBwUEdOHAg62JEycz+rZ3/I9P62s1UItd6yDQZbP/N2bBhg5544gn19PTo0KHS4JNbtmzRww8/rGXLSt993/72t7VmzRqZ2b+Z2TdUGmznfUlfc/efLPQa3ZZpK1hPwyPT8Mg0PDINr1GmdI8EAOTa+vXrtXfv3nOW33nnnZqamtLU1JTWrFlTWfwHKg0v/UlJX5T0t2a2KLXCAgDQBiptAIBcW7VqlZYsWdLswz8i6Yfu/q67vy7pVcU5oTcAAHOotAEACunBBx/UlVdeqQ0bNujNN9+sLL5A0q+qHjYjaUXqhQMAoAVRXNM23+DYj2sunx6/LuWSJKOV6y8kqZ3rL+YreqZZINPwyDS8bs30jjvu0D333CMz0z333KO77rpLjzzySL2H15yw1Mw2SdokSQMDA3PL62UqFT/XpJBpeGQaHpkmo1u/p1pFS1sGuP4CAJLV29urRYsW6bzzztNtt92m5557rnLXbyVdXPXQfklv1HoOd9/m7sPuPlw5oQYAQBaotGWA6y8AIFmzs7Nzf+/atUuXX3555eZbkm4xs983s0skDUl67pwnAAAgIgtW2szsETM7bmaHqpZtMbP/MLOp8s+aqvu+YWavmtkRM/tCUgUvIq6/AIDWjY6O6uqrr9aRI0fU39+v7du36+tf/7quuOIKXXnlldq/f78eeOCBysPfkfSYpF9I2ivpq+7+flZlBwCgGc1c0/Y9SQ9K+v685Q+4+/+uXmBml+l3XfmWS/qpmX2cL8SFJXn9BQAU2c6dO89ZtnHjxrqPd/d7Jd2bYJEAAAhqwZY2d/+ZpFNNPt9a0ZWvLVx/AQAAAKCWTq5p22xmB8vdJy8sL1uhJrvymdkmMztgZgdOnDjRQTGKgesvAAAAANTSbqXt7yR9TNJKSbOS7i8vtxqPrdmVr5tbhbj+AnmxYcMG9fT0VJ9E0JYtW7RixQqtXLlSK1eu1JNPPjl3H9e0AkBt7E8BdKKtedrc/VjlbzN7WNIT5ZszarIrXzfj+gvkxfr167V582bdeuutZy2/8847dffdd5+1jGtaAaC+VvanOnu6H/anANpraTOzvqqbN0qqjCy5R3TlAwqjxekpuKYVAOpguh8AnWhmyP+dkp6V9AkzmzGzjZL+2sxeNrODkv5M0p2S5O6HRVc+oPDqTE/R9DWtAIASpvsB0IxmRo8cdfc+d/89d+939+3u/n+5+xXufqW73+Dus1WPv9fdP+bun3D3p5ItPoC03XHHHXrttdc0NTWlvr4+3XXXXZW7mr6mtZsHIuK6FgAVDfantbA/BbpYJ6NHAuhCDaanaPqa1m4eiGj9+vXau3fvOcvvvPNOTU1NaWpqSmvWrKksrr6u5YuS/tbMFqVWWACJYrofAM2i0gagJQ2mp+Ca1iZwXQuACqb7AdCstkaPBNAdRkdHNTk5qZMnT6q/v19bt27V5OSkpqamZGYaHBzUQw89pEcffVTuftjMKte0nhHXtLbkwQcf1Pe//30NDw/r/vvv14UXXii1cF2LmW2StEmSBgYGki4ugBY1uz8te0fSLrE/BVBGpQ1AXUxPkY477rhD99xzj8xM99xzj+666y498sgj9R5ed+5LSdskaXh4uOZjAGSH/SmATtA9EgAyFuK6FgAAUFxU2gAgY1zXAgAAGqF7JACkiOtaAABAq6i0AUCKuK4FAMLYsGGDnnjiCfX09OjQoUOSpFOnTunmm2/W9PS0BgcH9dhjj1UGdpKZfUPSRknvS/qau/8ks8IDLaJ7JAAAAHKn1ryX4+PjWr16tY4eParVq1drfHy8chfzXiLXqLShMDZs2KCenp7q64F06tQpjYyMaGhoSCMjI3rzzTfn7jOzb5jZq2Z2xMy+kEWZASBG7E+RB7Xmvdy9e7fWrVsnSVq3bp1+9KMfVe76iJj3ckFs+/Gi0obCaOWMm5ldJs64AUBNtGAgr44dO6a+vj5JUl9fn44fP165q6V5L83sgJkdOHHiRJLFjQ7bfryotKEwWjzjtlaccQOAmmjBQJeoO++luw+7+/CyZcvSLlOm2PbjRaUNhdbgjNsKNXnGDQBACwbyobe3d24aldnZWfX09FTuYt7LNoXY9tE5Km3oVlZjWc0zbhxkAEDLaMFAJm644Qbt2LFDkrRjxw6tXbu2ctdbYt7LNHAslRAqbSi0BmfcZtTkGTcOMoC4ceF8OmjBQGxGR0d19dVX68iRI+rv79f27ds1Njamffv2aWhoSPv27dPY2Fjl4e9IekyleS/3inkvmxZi2+dYqnNU2lBoDc647RFn3IBC4ML5dNCCgdjs3LlTs7Ozeu+99zQzM6ONGzfqoosu0sTEhI4ePaqJiYmzrs9y93vd/WPu/gl3fyrDoucK234cqLRlgLPCyWjljJu7HxZn3IBC4ML58GjBALoT2368zs+6AN1o/fr12rx5s2699da5ZZWzwmNjYxofH9f4+Ljuu+8+6eyzwssl/dTMPs5Gca6dO3fWXD4xMVFzubvfK+neBIsEICOhBs2QtEmSBgYGEitrjNifAt2JbT9eC7a0mdkjZnbczA5VLVtiZvvM7Gj594VV99EqtADOCgNAVBg0AwAQtWa6R35PpX7/1cYkTbj7kKSJ8m0mLO4Aw6kCQDgMmgEAKJIFK23u/jNJp+YtXitpR/nvHZK+VLWcVqHkMZwqADTAhfMAgCJpdyCSXneflaTy78opTCYsbhPDqSJGDJqDPODCeeQB+1MAnQg9eiQTFreJs8KIUStDqdM9Gllh2G/kAVNTAOhEu5W2Y2bWJ0nl35ULsJiwuAmcFUZetDhoDt2jAaAOBiED0Il2h/zfI2mdpPHy791Vy//BzL6j0vD0tArVwHCqyLMGg+askPTPVQ+le3QNGzZs0BNPPKGenh4dOlQalPfUqVO6+eabNT09rcHBQT322GO68MLSoLxm9g1JGyW9L+lr7v6TzAoPICgGIQPQrGaG/N8p6VlJnzCzGTPbqFJlbcTMjkoaKd9mwmKgu9E9ugl0kQLQJvanQBdrZvTIUXfvc/ffc/d+d9/u7r9299XuPlT+farq8VwrABRYg0Fz6B7dBLpIAahgEDIAzQo9EAmAgmswaM4eMWhOW0J0keJsO5A/DEIGoFlU2gDU1cqgOXSPTk3NLlKcbQfixiBkADrR7kAkALoAg+ako9JFqq+vr+0uUgDixv4UQCdoaQOAjNFFCgAANEJLGwCkaHR0VJOTkzp58qT6+/u1detWjY2N6aabbtL27ds1MDCgxx9/vPLwdyTtUqmL1BnRRQoAgK5EpQ0AUkQXKQAA0Cq6RwIAAABAxKi0AQAAAEDEqLQBAAAAQMSotAEAAABAxKi0AQAAAEDEqLQBAAAAQMSotAEAAABAxJinDQAAAIUyODioxYsXa9GiRTr//NLhrpktkfSopEFJ05Jucvc3Mysk0AIqbegK1TtvSX8isfMGgHZwMIy82L9/v5YuXSpJMjNJGpM04e7jZjZWvv2X2ZUwX9j2s0WlDV2jsvM2s1fKi9h5A0AbOBhGTq2VdE357x2SJhVoPR0c+3HN5dPj14V4+miw7WeHShu6WWI7bwDoMhwMIypmpmuvvVZmpttvv72yuNfdZyXJ3WfNrCe7EhYGx1IpodKGrlC985a0tLyYnTdQcHTnCY+DYeTBM888o+XLl+v48eMaGRmRpA83+79mtknSJkkaGBjoqBw3P/SsHr396o6eIxZs+9mi0hYhrr8Kr3rn3dvb22Nmq5r931A775sfelaSCrPzBvKC7jxhxXIwDDSyfPlySVJPT49uvPFGHTx48EOSjplZX7ly0SfpeK3/dfdtkrZJ0vDwsKdV5tjFvO13wzFWR0P+m9m0mb1sZlNmdqC8bImZ7TOzo+XfF4YpanfZv3+/pqamJGn+9VdDkibKt9Gk6p23pLckXaXyzluSFtp5u/uwuw8vW7YslfICSNRalbrxqPz7S9kVJX/mHwxLmjsYltifInunT5/W22+/Pff3008/LUn/LWmPpHXlh62TtDuTAuYU2362QrS0/Zm7n6y6zRnMZNBnuE2nT5/WBx98oMWLF+v06dOS9EeSDul3O+9xsfNuGS3CyINOuvN0ema4iGd+5+9PaxwMsz9tA914wzp27FilUqEzZ87oK1/5ip599tnfqLR+PmZmGyX9u6QvZ1jMXGHbz14S3SOpXHSo3euvOMCobf7OW9Jb7r7XzJ4XO++OMCInYtdJdx66SJ2Lg+Hk0I03nEsvvVQvvfTSWcu+9a1vyd1/LWl1NqXKN7b97HVaaXNJT5uZS3qo/AWXyhnMiiJWNNq9/ooDjNrm77zN7D8lsfNOBidtOsDZ9vA6ua4F5+JgOFXsTxENtv3sdVpp+5y7v1GumO0zs39p9h+pYNTX6PorDjIQC0bkTAZn28OhOw/yglH5ACyko4FI3P2N8u/jknaphcEdUNv8i2d17vVXEgcZiMAzzzyjF198UU899ZQktTwip5kdMLMDJ06cWPDxlRb1LsWgGW06duyYPv/5z+tTn/qUrrrqKl133XWSVOnOM2JmRyWNlG8Dmanen373u9+VWhyVr5X9qVTap3b5fhXInbZb2szsQ5LOc/e3y39fK+n/EWcwO8L1V8iLTlqEaWmvLc1BM4rYtXy+LLrz1JoIenDsx0wEjYYYnh7AQjrpHtkraVe5+875kv6BykXnuP4KecCInMlg0Ayg+9CNF0Az2q60ufsvJX2qxnIqF0DB0SKcDAbNALoPo/IBaEYSQ/4DXa1W96iKz16yJMWSJIcW4fA42w50J0blA9AMKm0AEAHOtgMAgHqotAFABDjbDgAA6uloyH8AAAAAQLKotAEAAABAxKi0AQAAAEDEuKaty9WbCFYSk8ECAAAAEaDSBgBd6Oevn5JU+8QNJ2wAAFlrNIVSN35PUWkDAAC5wwEdgG5CpQ1I0c9fP0XLRhvq5SaRHQC0ql5LO/vT8Pj+Qii5qrTVW+k/e8mSlEsCAAAAIAuNWtqLKleVNgAA8ogz7QCATlBpAwAAAFBYRbgGlkob6uLMMAAAAPKiyN0mC1Fp4yJP5F0RzgABAAAgGYWotDVCZQ4AECtO2CSD7/72sD4C8Sp8pQ3IOw4+AAAAuhuVNgAAADTECcTw2rn+irzTE1vLc2KVNjP7oqS/kbRI0v9x9/GkXqtbxJJpkXYysWRaJGQaXtqZdsPBGetpeHnINLaDsIXkIdM8ItfwyDR5iVTazGyRpO9KGpE0I+l5M9vj7r9I4vXakcMdd/SZNhLjQWDeM41RFpnGuG6FFNN6mrf9Zj0xZdpIntbtvGSaJ2SaDHINL++ZhhxxMsnvyaRa2q6S9Kq7/1KSzOyHktZKKuyHl8KXaK4zjVSuM03rgK7F14km00i343ZEk2mBkGl4ZBoemSYj17lG+t2W60zbkcXUAklV2lZI+lXV7RlJn03otaLQzgF0i//TdZm2g0zbO8sTcOeT60xD7oS7bdsPeSAReh2u8T+5yLSeGOYhKlqmUjonwoq4nqa1PgbcX+Qi15BS+G7rukzb0ek+JqlKm9VY5mc9wGyTpE3lm++a2aGEypIpu6/h3UslnVzgfz5aWVzj/xtl+l9mdqTR68RggXyqdfQe2s1UqpnrrzspS8rOyq2FvBcUONMjing9bcHce2gn6xr/81EVONNAGTX7P5UsOsk0t9t+aDW2f9bT2pp+b12SadvlCrC/6PR4KtZMUzUv009UFtd46EKZ5ml/2oxQ6/ZH6zwssUrbjKSLq273S3qj+gHuvk3SNkkyswPuPpxQWaLV4vtuKdMOXidKCb2HBTOVzs01T3lmUNa2MpXylWs9SbwHM7taXZxpKNVZtJtpnvJMu6ysp7V18t6KmGkk5WrreCqSskfFzA6U/2w506Llmcb7OS+h531e0pCZXWJmF0i6RdKehF6rW5BpeGQaHpmGR6bhkWl4ZBoemSaDXMMj0xQk0tLm7mfMbLOkn6g09Ocj7n44idfqFmQaHpmGR6bhkWl4ZBoemYZHpskg1/DINB2JzdPm7k9KerLJh5/Tpa9LtPS+W8y07deJVCLvoc1M85Rn6mVlPQ2vyzMN5aws2PbDYz2tqaP3VsBMoyhXF2z/aZnLpI1Mi5Zn4u/H3M+5phUAAAAAEImkrmkDAAAAAASQeaXNzL5oZkfM7FUzG8u6PEkws4vNbL+ZvWJmh83sz8vLl5jZPjM7Wv59YeDXzW22ZjZtZi+b2VRldKKk82qiTNHmmdU6FkLMuTYS4zpaVbZcZhpK6M8m5jzzvO1LcWfbqhj2CTHl2WDd3GJm/1HOacrM1mRZzmbElGtWQq3fec8y032uu2f2o9LFiq9JulTSBZJeknRZlmVK6H32Sfp0+e/Fkv5V0mWS/lrSWHn5mKT7yHau/NOSls5bllheec8zi3WsG3JdoOxRraNFyDTGzyb2PPO67ech2zbeT6b7hNjybLBubpF0d9afV15zzTCHjtfvImSZ5T4365a2qyS96u6/dPffSvqhpLUZlyk4d5919xfLf78t6RWVZo9fK2lH+WE7JH0p4MsWMdsk81pI1HlmtI6FEHWubYgh76JlGkq7n03UeeZ425cizzaQND+HqPJssG7mTVS5RqbV9Tv3WWa5z8260rZC0q+qbs8onxt008xsUNKfSvq5pF53n5VKK4GknoAvlfdsXdLTZvaCmW0qL0syr4XkJs8U17EQcpNrDbGtoxV5zjSUkJ9NbvLM2bYv5SjbJmW9T4g2z3nrpiRtNrODZvZIrF13q0Sba8pCrN+FyjLtfW5iQ/43yWosK+xwlmb2YUn/KOkv3P03ZrXefriXq7EsT9l+zt3fMLMeSfvM7F8yLk8u8kx5HQshF7nWEds6WpHnTEMJ+dnkIs8cbvtSTrJtQdb7hCjzrLFu/p2kv1KpbH8l6X5JGzIs4kKizDUDIdbvwmSZxT4365a2GUkXV93ul/RGRmVJlJn9nkof7t+7+z+VFx8zs77y/X2Sjgd8yVxn6+5vlH8fl7RLpSb1JPNaSPR5ZrCOhRB9rvVEuI5W5DbTUAJ/NtHnmdNtX8pBtq2IYJ8QXZ611k13P+bu77v7B5IeVimnmEWXaxYCrd+FyDKrfW7WlbbnJQ2Z2SVmdoGkWyTtybhMwVmp+r1d0ivu/p2qu/ZIWlf+e52k3QFfNrfZmtmHzGxx5W9J10o6pGTzWkjUeWa0joUQda71RLqOVuQy01AS+GyizjPH274UebatiGSfEFWe9dbNyoFt2Y0q5RSzqHLNQsD1O/dZZrrPDT2ySas/ktaoNPLKa5K+mXV5EnqPn1ep+fegpKnyzxpJF0makHS0/HsJ2bpUGlXopfLP4UrZk84rz3lmtY4VPdcGZY5yHc1zpjF/NjHnmedtP/ZsW3wfUewTYsqzwbr5A0kvl5fvkdSX9eeXp1wzev/B1u+8Z5nlPtfKBQAAAAAARCjr7pEAAAAAgAaotAEAAABAxKi0AQAAJKw8J9lxMztUtWyLmf2HmU2Vf9ZU3fcNM3vVzI6Y2ReyKTWAWERxTdvSpUt9cHAw62JE6YUXXjjp7sta/T8yra/dTCVyrYdMk8H2Hx6ZhkemzXn77be1aNEivf766/rkJz8pSXrjjTd03nnn6Y//+I/PeuwLL7zwpkqTEF8labmkn0r6uLu/3+g1ui3TVrCehkem4TXKNOvJtSVJg4ODOnDgQNbFiJKZ/Vs7/0em9bWbqUSu9ZBpMtj+wyPT8Mi0edPT07r++uvn3veWLVv04Q9/WHffffdZjzOz/0/SD939XUmvm9mrKlXgnm30/N2YabNYT8Mj0/AaZUr3SAB1bdiwQT09Pbr88svnlm3ZskUrVqzQypUrtXLlSj355JNz99GdBwBa8+CDD+rKK6/Uhg0b9Oabb1YWX6BSS1vFjKQVqRcOQDQWrLTRBxvoXuvXr9fevXvPWX7nnXdqampKU1NTWrOmtPmb2WUqTZT5SUlflPS3ZrYozfICQJ7ccccdeu211zQ1NaW+vj7dddddjR5e83oWM9tkZgfM7MCJEyeSKSiAzDXTPfJ7kh6U9P15yx9w9/9dvWDeQdtyST81swX7YM83OPbjmsunx69r5WlQhUzD64ZMV61apenp6WYfvlZtdOep1g2Zpo1Mw6uXqUSu7erWTHt7e+f+vu2223T99ddXbv5W0sVVD+2X9Eat53D3bZK2SdLw8PBcxa5bM00SmSaD76nmLNjS5u4/k3SqyeebO2hz99clVQ7aABRIne48K0R3HgBo2uzs7Nzfu3btqu6K/pakW8zs983sEklDkp5LvYCRows/ukkn17RtNrOD5e6TF5aXcdAGFFyD7jxW4+F05wEASaOjo7r66qt15MgR9ff3a/v27fr617+uK664QldeeaX279+vBx54oPLwdyQ9JukXkvZK+mqrvZa6QStd+CX9gejCjxxrd/TIv5P0VyodkP2VpPslbVCLB22SNknSwMBAm8UAkLYG3Xlm1GF3HgAoqp07d56zbOPGjXUf7+73Sro3wSLlXotd+D8i6buddOEHstRWS5u7H3P39939A0kP63ddIFs6aHP3YXcfXrasremdAGSgQXeePaI7DwAgY4zIiSJqq9JmZn1VN2+UVBlZkoM2ZIa+7eG10p3H3Q+L7jwAgAwxIieKasHukWa2U9I1kpaa2Yyk/ynpGjNbqdLKPi3pdql00GZmlYO2M+KgDSlav369Nm/erFtvvfWs5XfeeWetiUuDjHRadHTnAQDkSZIjcgJZWrDS5u6jNRZvb/B4DtqQibSHpwcAAHGZnZ1VX1+pQ1idETm/o9LJWnqDIVfaHYgEyI0HH3xQ3//+9zU8PKz777+/sniFpH+ueljdvu0MmgMAQHxGR0c1OTmpkydPqr+/X1u3btXk5KSmpqZkZhocHNRDDz1Uefg7knaJ3mDIKSptKLQ77rhD99xzj8xM99xzT1vD09NNAgCA+NCFH92kk3nagOj19vZq0aJFOu+883TbbbfpuefmekI0PdIpAAAAkCUqbSg0hqcHAABA3lFpQ2EwPD0AhFFrCpVTp05pZGREQ0NDGhkZqZ7/iilUACBhVNpQGDt37tTs7Kzee+89zczMaOPGjfrBD36gl19+WQcPHtSePXvmRpSSSn3b3f1j7v4Jd38qw6IDQFTWr1+vvXv3nrVsfHxcq1ev1tGjR7V69WqNj49X7voD/W4KlS9K+lszW5RqgQEEwQmbeFFpAwAAZ1m1apWWLFly1rLdu3dr3bp1kqR169bpRz/6UeWuj6g8hYq7vy6pMoUKgJzhhE28qLQBQIpqncXcsmWLVqxYoZUrV2rlypV68skn5+7jLObCODOcjmPHjs31Vujr69Px48crd10g6VdVD607hQqAuHHCJl5U2gAgRbXOYkrSnXfeqampKU1NTWnNmjWVxZzFbAJnhqNUc3oUM9tkZgfM7MCJEyfSLhOANnDCJg5U2jLAWWGge9U6i9nAR8RZzAVxZjgdvb29cyPyzs7Oqqenp3LXb9XkFCruvs3dh919eNmyZUkWF0A2OGGTECptGeCsMID5HnzwQV155ZXasGFD9UkbzmK2KcSZYQ4yznbDDTdox44dkqQdO3Zo7dq1lbveElOoAIXFCZs4UGnLAGeFAVS744479Nprr2lqakp9fX266667Gj2cs5jh1cy0mw8yak2hMjY2pn379mloaEj79u3T2NhY5eHviClUgMLihE0czs+6ACihvzDQvXp7e+f+vu2223T99ddXbrZ0FlPSNkkaHh6uWQnpJpUzw319fW2fGe5mO3furLl8YmKi5nJ3v1fSvQkWCUAKRkdHNTk5qZMnT6q/v19bt27V2NiYbrrpJm3fvl0DAwN6/PHHKw9/R9IulU7YnBEnbBJFpS2f6p5pl7RJkgYGBlItEID2VSoXkrRr167q613fUuks5nckLRdnMZtWOTM8NjZW78wwmQLAPJywiReVtkiEOCvMmXYgfrXOYk5OTmpqakpmpsHBQT300EOVh3MWswmcGQYAFB2VtkhwVhjoDrXOYm7cuLHu4zmLuTDODAMAio5KWwY4KwwAAACgWVTaMsBZYQAAusuGDRv0xBNPqKenR4cOHZJUmqP15ptv1vT0tAYHB/XYY4/pwgsvlFSao1XSRknvS/qau/8ks8IDyBxD/gMAACSMOVoBdIJKG4C6NmzYoJ6enurRDHXq1CmNjIxoaGhIIyMj1RNBy8y+YWavmtkRM/tCFmUGgBgxRyuATixYaTOzR8zsuJkdqlq2xMz2mdnR8u8Lq+7joA0oiFbODJvZZeLMMAA0jTlaATSrmZa276l0AFZtTNKEuw9Jmijf5qANKJgWzwyvFWeGASApdedoNbMDZnbgxIkTaZcpU/QGQTdZsNLm7j+TdGre4rWSdpT/3iHpS1XLOWgDCqzBmeEV4swwADStMkerpI7maHX3YXcfXrZsWZLFjQ7XCaKbtHtNW6+7z0pS+XdlL8NBG9C9rMYyzgwDQB2VOVol1Zuj9ffN7BIxR2tNXCeIbhJ6IBIO2pAZukmko8GZ4RlxZhgAahodHdXVV1+tI0eOqL+/X9u3b9fY2Jj27dunoaEh7du3T2NjY5WHvyPpMZXmaN0r5mhtGtcJoqjarbQdM7M+SSr/rmwRHLQhMwyakY4GZ4b3iDPDAFDTzp07NTs7q/fee08zMzPauHGjLrroIk1MTOjo0aOamJg4q9XI3e9194+5+yfc/akMi15kNCwgN9qttO2RtK789zpJu6uWc9CGTDBoRnitnBl298PizDAAIENcJ4iiOn+hB5jZTknXSFpqZjOS/qekcUmPmdlGSf8u6ctS6aDNzCoHbWfEQRsytsCgGf9c9dC63STMbJOkTZI0MDCQWFljtHPnzprLJyYmai5393sl3ZtgkQAAqKvSG2RsbKzedYLfkbRcNCwgZxastLn7aJ27Vtd5PAdtyIOmr790922StknS8PBwzccAAIB0jY6OanJyUidPnlR/f7+2bt2qsbEx3XTTTdq+fbsGBgb0+OOPVx7+jqRdomEBObVgpQ3Is0o3ib6+vrluEm+99ZbUwvWXAAAgPvQGQTcJPXokEBUGzQAAAEDeUWlDYTBoBvKAqSkAAECrqLShMBhOGXnQytQUkv5ATE0BAEDXo9IGAClqcWqKj4ipKQAA6HpU2gAgYw2mprhA0q+qHtpwagomgwUAoJiotAFAvtSdmoLJYAEAKCYqbQCQscrUFJLmpqYo+62YmgIAgK5HpQ0AMtZgaoq3xNQUAAB0PSbXBoAUjY6OanJyUidPnlR/f7+2bt2qsbEx3XTTTdq+fbsGBgb0+OOPVx7+jqRdKk1NcUZMTQEAQFei0gYAKdq5c2fN5RMTEzWXu/u9ku5NsEgAACBydI8EAAAAgIjR0gYAAJo2ODioxYsXa9GiRTr//NJhhJktkfSopEFJ05Jucvc3MyskgODY9rNFpQ0AUFgcZCRj//79Wrp0qSTJzCRpTNKEu4+b2Vj59l9mV0IASWDbzw6VtghVH2RI+hOJgwwAaBcHGalYK+ma8t87JE2KTIFuwLafEiptkaocZJjZK+VFHGQAQBgcZHTAzHTttdfKzHT77bdXFve6+6wkufusmfXU+d9NkjZJ0sDAQJDy3PzQs5KkR2+/OsjzAaitk20fnaPSlh8cZABAizjICO+ZZ57R8uXLdfz4cY2MjEjSh5v9X3ffJmmbJA0PD3tCRQSQgE62/VAnbLr5JA2VtghVH2RIWlpezEEGALQohoOMiqIcbCxfvlyS1NPToxtvvFEHDx78kKRjZtZX/n7qk3Q800ICCK6TbZ8TNp1jyP8IPfPMM3rxxRf11FNPSVKPma1q5v/MbJOZHTCzAydOnOioDDc/9OzcAQYA5NX8gwxJcwcZkrTQQYa7D7v78LJly1IqcdxOnz6tt99+e+7vp59+WpL+W9IeSevKD1snaXcmBQSQCLb97NHSFqHqgwxJb0m6Sk2cyeAsBgD8zunTp/XBBx9o8eLF9Q4yxsVBRkuOHTtWqfzqzJkz+spXvqJnn332Nypl+ZiZbZT075K+nGExc4dRThE7tv3sdVRpM7NpSW9Lel/SGXcfZifTmfkHGZL+SNIhcZCByDDKKWKXxUHG4NiPay6fHr8u1Etk6tJLL9VLL7101rJvfetbcvdfS1qdTamKgVFOETO2/eyFaGn7M3c/WXWbnUwH5h9kSHrL3fea2fPiTAYiwyiniBkHGcg5BiDrAK2XKJokukeyk+nA/IMMM/tPSRxkdIhWodSw/QNAi7KaRqEog+PUQ+sliqTTSptLetrMXNJD5WuqGOUQUaJVKKxORjlt9yCj6AcYALoT0yikhhOLyK1OK22fc/c3ygdm+8zsX5r9R+ZrQATYeXeg+iCjt7e36VFOJQ4yAKAa0yiEl3brJcejSFpHlTZ3f6P8+7iZ7VKToxyW/4eDNqQmi1ahiqLuyNsd5RT1cQ0G0H0Y5TQZtF6iaNqutJnZhySd5+5vl/++VtL/I3YyuVJvpDNJ+uwlS1IsSbJoFQqLUU6TwzUYwNmKPiInQ6kng9bL/Gt0jNqNOmlp65W0q3xQcb6kf2CUQ8SKVqGwGOU0VXTjBQqMUU7Do/USRdR2pc3dfynpUzWWs5NBVGgVCo9RTpPRyTUYAIASWi9RREkM+Q9EhVYh5EUn12CEGtwJAPKO1ksUEZU2FB6tQsiLTq7B4NpLAACK67ysCwAAKHXjffvtt+f+rnENhkQ3XgAAuhItbQAQAa7BAJCW+aPyVW4XZUROoIiotAFABLgGAwDy6+evn5J0boWYijBCodKGun7++qmac2SwAwIAAADSwzVtAAAAABCxQrS01WsRkmgVAgAAAJBvhai0AQAAAOgO3dhYQ6UNQPTqjXTWSJF33AAAoLtQaQMCq1ehGBz7MRUJAAAAtCxXlbZmzq4DABCb6u+v6r85kQMAaEauKm0AAAAL6cbrXQAUG5U2AAAy0qgHCRUMAEAFlTYA6EJUFgAAyA8qbQAAIHfaGVUWAPKKShtaxrUCAAAAQHqotAEAgK5At+D2ccIWeVDk9ZRKG5CiIu9MAAAAkAwqbUAEOPsbHpkCALLGdxFCSazSZmZflPQ3khZJ+j/uPp7UazVSpJaNWDItEjINj0zDI9Pw8pBp3r6/8pBpI60OZJLG55B2pt0ymEve19UYxZ5pO+t2o208i/1zIpU2M1sk6buSRiTNSHrezPa4+y+SeL125O3MB5mGl4dM84ZMw8si07xVFlqV9/U0xn1t3jMNLcQBYhEyjXFfElOuMebTjpgyDSm2kxhJtbRdJelVd/+lJJnZDyWtlVTYDy+FDSzXmYZc8QNmnYtMsz772+KXSi4yzRkyDa+wmWZ4EFjYTOtJ4YAuF5nGdmDbhFzkWk/oFqNAcp1pXiRVaVsh6VdVt2ckfbb6AWa2SdKm8s3/MrMjgcuwVNLJdv7R7mv9ORv8TzMalfWj5d/tZtp2DilrupwdZi21kKmUyrparePPq4V8Onqtea/Taaa/7qQsrWoyoxi2nY+qs/U06HsIsO11KsT7aTfTd83sUIevnZk2P7tm8+50PQ0hie019HM2fL4a+9Rotv0AzipPRvuS3BxPhc4nwPPVe++dZJrq936HsjhGPUdSlTarsczPuuG+TdK2hF5fZnbA3Ydjf84WnretTJMqc2gZlXPBTKXk19VqaeaQ0Gu1lWmM62ksZTKzL9dY3NR6Gst7CCXU+2kn06Jl2YxW3nMn62kIefjOb/X5irTtR1aeQh9PJaGJ995ypnnKM5aynpfQ885Iurjqdr+kNxJ6rW5BpuGRaXhkGh6Zhkem4ZFpeGSaDHINj0xTkFSl7XlJQ2Z2iZldIOkWSXsSeq1uQabhkWl4ZBoemYZHpuGRaXhkmgxyDY9MU5BI90h3P2NmmyX9RKWhPx9x98NJvFYDSXRnS6qL3ILP20GmqXTrCyD1ckayns6XZg7BX6tg62kUZepwPY3iPQQU5P20mWnRsmxG0+85gv1pHr7zW3q+gm370ZSnYN9TaWn43rtgnxpFWc39nO7RAAAAAIBIJNU9EgAAAAAQAJU2AAAAAIhY4SptZjZtZi+b2ZSZHejgeR4xs+PV8/KY2RIz22dmR8u/LwzwnFvM7D/K5Z0yszXtlnnea33RzI6Y2atmNhbiOUMxs4vNbL+ZvWJmh83sz8vLE8kiL0Ktu3WeO/j6HLBsma6rDdbHKPJpR9aZhlBre8jqMylCngvJ83bQ6b4z9P4xze/6Gq+d+bqa53WpnhhyTVPS+9+Y84x6/XX3Qv1Impa0NMDzrJL0aUmHqpb9taSx8t9jku4L8JxbJN0dOINFkl6TdKmkCyS9JOmyrD+bqvL1Sfp0+e/Fkv5V0mVJZJGnn1Drbp3nDr4+BypX5utqg/Ux83zymmmg93HO9pDFZ1KUPJt4n7ndDjrdd4beP6b1XV/jdaNYV/O8LsWca8rvObH9b+x5xrz+Fq6lLRR3/5mkU/MWr5W0o/z3DklfCvCcSbhK0qvu/kt3/62kH6pU9ii4+6y7v1j++21Jr0hakW2pii2J9TmQzNfVButjDPm0I/NME5TFZ1LkPOcUcDtoWuj9Y4rf9fNFsa4WcF2KItcIhPr8os4z5vW3iJU2l/S0mb1gZpsCP3evu89KpQ9VUk+g591sZgfLXSpCNLeukPSrqtszirRSZGaDkv5U0s/Li0JnkSdJrru1JLU+tyKqdXXe+hhDPu2IKtMO1NoesvhMipJn03K4HSSx70zifSf9/RbduprDdamW6HJNQZL739zkGdv6W8RK2+fc/dOS/oekr5rZqqwLtIC/k/QxSSslzUq6P8BzWo1l0c3tYGYflvSPkv7C3X+jZLLIk7ytuyFEs67WWB/zKppMOxTL9lCUPJuS0+0glnWlkTS+36JaV3O6LtUSVa4pSXKbykWeMa6/hau0ufsb5d/HJe1SqRk2lGNm1idJ5d/HO31Cdz/m7u+7+weSHlaY8s5Iurjqdr+kNwI8bzBm9nsqbQx/7+7/JCWWRW4kvO7WEnx9bkMU62qt9VFx5NOOKDLtVJ3tIYvPpBB5NiOv20FC+86g7zul77do1tW8rkt1RJNrWhLe/0afZ6zrb6EqbWb2ITNbXPlb0rWSDjX+r5bskbSu/Pc6Sbs7fcLKClB2o8KU93lJQ2Z2iZldIOkWlcoeBTMzSdslveLu36lankQWuZDCultL8PW5DZmvq/XWR8WRTzsyz7RTDbaHLD6T3OfZjLxuBwnuO4O+75S+36JYV/O6LjUQRa5pSWH/G3WeUa+/aY98kuSPSiPRvFT+OSzpmx08106VujC8p9JZgY2SLpI0Ielo+feSAM/5A0kvSzqo0grRFyiLNSqNePNaJzkk9Dl9XqWm8IOSpso/a5LKIg8/IdfdOs8ffH0OWLZM19UG62MU+eQx0wDlr7k9ZPWZ5D3PJt9jLreDEPvO0PvHNL/ra7x25utqXtel2HNN8b0mvv+NOc+Y118rFxAAAAAAEKFCdY8EgNhZixPvmtk3yhOQHjGzL2RTagAAkCVa2gAgReVRuP5L0vfd/fLysi2S/svd//e8x16mUlerqyQtl/RTSR939/dTLTQAAMhUFJW2pUuX+uDgYNbFiNILL7xw0t2Xtfp/ZFpfu5lK5FoPmbbm3Xff1auvvqpPfvKTkqQ33nhD5513nv74j//4rMe98MILpyXd6+7/P0kys59I2uLuzzZ6/m7MtFnsU8Mj0/DINDwyDY9Mw2uU6flpF6aWwcFBHThwIOtiRMnM/q2d/yPT+trNVCLXesi0NdPT07r++uvn3veWLVv0ve99T7/97W81PDys+++/XxdeeKHM7P9Tk5OQlidA3SRJAwMDXZdps9inhkem4ZFpeGQaHpmG1yhTrmkDgIzdcccdeu211zQ1NaW+vj7dddddjR5es3uEu29z92F3H162rK1GTwAAECkqbQCQsd7eXi1atEjnnXeebrvtNj333HOVu36ryCchBQAAyaPSBgAZm52dnft7165duvzyyys335J0i5n9vpldImlI0nPnPAEAACi0KK5pm29w7Mc1l0+PX5dySYqDTMMj0/C6IdPR0VFNTk7q5MmT6u/v19atWzU5OampqSmZmQYHB/XQQw9VHv6OpF2SfiHpjKSvtjpyZDdkumHDBj3xxBPq6enRoUOlmRS2bNmihx9+WJWuot/+9re1Zk1pJgUz+4ZKEx6/L+lr7v6TVl6vXqZSsXJNE5mGR6bhkWkyuuF7KoQoK20AUFQ7d+48Z9nGjRvrPt7d75V0b4JFyr3169dr8+bNuvXWW89afuedd+ruu++e//A/kHSLpE+qPI2CmTGNAgAganSPBADk2qpVq7RkyZJmH/4RST9093fd/XVJr6o0Dx4AANGi0gYAKKQHH3xQV155pTZs2KA333yzsvgCNTmNAgB0mw0bNqinp6f62mpt2bJFK1as0MqVK7Vy5Uo9+eSTc/eZ2TfM7FUzO2JmX8iizN2CShsAoHBCTKNgZpvM7ICZHThx4kQyBQWAiKxfv1579+49Z/mdd96pqakpTU1NzV0frLO7m39R0t+a2aLUCttlqLQBAAonxDQKzH0HoNvQ3TxeVNoyQNNzMsgVQAXTKABAOHQ3zx6VtgzQ9JyMVnI1s8tErkAhjI6O6uqrr9aRI0fU39+v7du36+tf/7quuOIKXXnlldq/f78eeOCBysPfkfSYStMo7FUb0ygAQDehu3kcGPI/A6tWrdL09HSzD/+IpO+6+7uSXjezStPzs8mULr9azHWtyk36Ilcg15hGAQCS09vbO/f3bbfdpuuvv75ys6Xu5pK2SdLw8HDNih0ao6UtIjQ9J6NOritErgAAAA3R3TwOVNoiQdNzMhrkajUeTq4AAKBr0d08XlTaIsFIZ8lokOuMyBUAkBIGywqPTMPbuXOnZmdn9d5772lmZkYbN27UD37wA7388ss6ePCg9uzZo76+vrnHu/u97v4xd/+Euz+VYdELj0pbJGh6TkaDXPeIXAEAKWEQsvDIFN2EgUgyMDo6qsnJSZ08eVL9/f3aunWrJicnNTU1JTPT4OCgHnroocrD35G0S6Wm5zOi6bmuZnN99NFH5e6HzazSpE+uAIBEMQhZeGSKbkKlLQOMdJYMcgUA5M2DDz6o73//+xoeHtb999+vCy+8UGIQso6QKYqI7pEAAAAZYBCy8MgURUWlDQAAIAMMQhYemaKoqLQBqKvWyFynTp3SyMiIhoaGNDIyUj33HSNzAUALGIQsPDJFUXFNG4C61q9fr82bN+vWW2+dWzY+Pq7Vq1drbGxM4+PjGh8flySZ2WX63chcyyX91Mw+zgAvAMAgZEkgU3QTKm0A6qo1Mtfu3bs1OTkpSVq3bp2uueaayl1rJf2QkbkA4FwMlhUemaKb0D0SQEuOHTs2N7FmX1+fjh8/XrlrhZocmYuLvIG40TUaAOJCpQ1AKFZjWc2RubjIG4hbrUmLK12jjx49qtWrV891jRaTFgNA4hastJnZI2Z23MwOVS1bYmb7zOxo+feFVfdxtg0osN7e3rkLvWdnZ9XT01O5a0ZNjswFIG6rVq3SkiVLzlq2e/durVu3TlKpa/SPfvSjyl0fUblrtLu/LqnSNRoAEEgzLW3fU+nMWbUxSRPuPiRponx7/kAEnG0DCuiGG27Qjh07JEk7duzQ2rVrK3ftESNzAYXVoGt005MW0zUaANqzYKXN3X8m6dS8xWsl7Sj/vUPSl6qWc7YNKIjR0VFdffXVOnLkiPr7+7V9+3aNjY1p3759Ghoa0r59+zQ2NiZJcvfDkh5TaWSuvWJkrpq4Vghdgq7RABBQu9e09br7rCSVf1f6RzU9EAGA+O3cuVOzs7N67733NDMzo40bN+qiiy7SxMSEjh49qomJibO6ULn7ve7+MXf/hLs/lWHRo8W1QsirBl2jm560GADQntADkTQ9EAFdJAB0I64VQl416Br9lugaDQCJarfSdszM+iSp/LvSsb3pgQjoIgEAJVwrhNi00jVapUmL6RoNAAlqd3LtPZLWSRov/95dtfwfzOw7kpaLs20AEFrda4UkbZOk4eHhmo8BmlVr0mJJmpiYqLmcSYsBIFnNDPm/U9Kzkj5hZjNmtlGlytqImR2VNFK+zUAEANAGrhXqDIO7AACKrpnRI0fdvc/df8/d+919u7v/2t1Xu/tQ+fepqsczEAEAtIBrhTrD4C4AEAYnweIVeiASAEADXCsUHoO7AEAYnASLV7vXtAEA2sC1QukINbiLpE2SNDAwkFhZASAWq1at0vT09FnLdu/ercnJSUmlk2DXXHON7rvvPql0Euy77v6upNfNrHIS7NkUi9w1aGnLAE3PySBXAG1iImgAqCPESTB0jkpbBmh6TkYruZrZZSJXoLAY3AUAMsH8zAmh0pYBrr9IRou5rhW5AoXF4C4AEEaIk2D0XOgclbZIMLluMhrkukLkChQCg7sgD+jCHx6ZpoOTYHGg0pZPXH/ROauxjFyBHNq5c6dmZ2f13nvvaWZmRhs3btRFF12kiYkJHT16VBMTE2e1wjM1DbLApRHhkWl4nASLF6NHRqLS9NzX18f1FwHVyvWtt96SSi1r5AoASAWj8oVHpuExwnG8aGmLBE3PyWiQ6x6RKwAgQ1waER6ZoqiotGWApudktJKrux8WuQIA8oMu/OGRKXKD7pEZoOk5GeQKAMgTLo0Ij0xRVLS0AQAAZIBLI8IjUxQVLW0AAAAJGx0d1eTkpE6ePKn+/n5t3bpVY2Njuummm7R9+3YNDAzo8ccfrzz8HUm7VOrCf0Z04a+JTNFNqLQBAAAkjC784ZEpugndIwEAAAAgYlTaAAAAACBiVNoAAAAAIGJU2gAAAAAgYlTaAAAAACBiVNoAAAAAIGIM+Q+gLYODg1q8eLEWLVokSX8iSWa2RNKjkgYlTUu6yd3fzKqMAAAARUClDUDb9u/fr6VLl8rMXikvGpM04e7jZjZWvv2X2ZUQQGjVJ2zOP790GMEJGwBIFt0jAYS0VtKO8t87JH0pu6IASMr+/fs1NTWlAwcOVBZVTtgMSZoo3wYABNJRS5uZTUt6W9L7ks64+zBn24DuYGa69tprZWaStLS8uNfdZyXJ3WfNrKfO/26StEmSBgYGOi7LzQ89K0l69ParO36uLNGCgRxbK+ma8t87JE2KVnYACCZE98g/c/eTVbfpHgV0gWeeeUbLly/X8ePH1dvb22Nmq5r9X3ffJmmbJA0PD3tihcyhSpdTSZUKMftURKX6hM3tt99eWdzUCRsAQHuSuKaNs21AF1i+fLkkqaenR5LeknSVpGNm1lc+aOuTdDyzAhYH+1REpfqEzcjIiCR9uNn/DdHKXpSWdQBoRaeVNpf0tJm5pIfKZ88T7x7FDhvI1unTp/XBBx9o8eLFOn36tCT9kaRDkvZIWidpvPx7d3alzB9aMMKjy2l41SdsbrzxRh08ePBDavKEDa3sANCeTittn3P3N8oHEfvM7F+a/Ud23EB+HTt2TDfeeKMk6cyZM5L0lrvvNbPnJT1mZhsl/bukL2dXyvzJugWjomgnxuhyGs78EzZPP/20JP23OGEDFB4nwbLVUaXN3d8o/z5uZrtE96ggmP8qPDIN69JLL9VLL700d9vM/lOS3P3XklZnVa68owUjNXQ5bdP8EzZf+cpX9Oyzz/5GpcoaJ2yAguMkWHbaHvLfzD5kZosrf0u6Vmd3j5I429a2ynDKkubPf8Vwym0iU8Ts9OnTevvtt+f+rtGCIbFPbVmly+lnPvMZbdu2rbL4rC6nkuhy2qTKCZuXXnpJhw8f1je/+U1JpRM27r7a3YfKv09lXFQA6WCqn5R00tLWK2lXuZZ9vqR/oHtUojgzHB6Z5sTg2I/r3jc9fl2KJUkOLRjJSLvLadG6liIddDsLj0zD6+S6a/annWu70ubuv5T0qRrLE+seNf/ArXK7KAdtFZ3Mf4XayBSxm9/lVJK+9a1v0eW0Q3Q5RV7Q7Sw8Mg2rk5Ng7E8713b3SCTnmWee0YsvvqinnnpKkpqe/8rMNpnZATM7cOLEiaZeq3IWo+jazVRqP9duyRaIFV1OkXN0OwuPTDsw/ySYpLmTYJLEWBbJotIWoUbzX0n1Nwp33+buw+4+vGzZstTKmwftZiqRK5BXx44d0+c//3l96lOf0lVXXaXrrrtOkipdTkfM7KikkfJtIDOdXHvZzonFbkCmYXESLHtJTK6NDjD/VXhkCnQnupwWQ6NrWouCbmfhkWlYXHedPSptkWH+q/DIFAAQs06uvURtZBoWJ8GyR6UtMsx/FR6ZAudq1Hrx2UuWpFgSoLsxYXl4ZIoiotIGAACQEbqdhUemKCIqbQAAABmh21l4ZIoiYvRIAAAAAIgYlTYAAAAAiBiVNgAAAACIGJU2AAAAAIgYlTYAAAAAiBiVNgAAAACIGEP+d7mfv36q7iS70+PXpVwaoHXV62/136y/AADk189fPyVJNY9Tu/E7nkobAACB1DvI6MYDjKTVO+EokTeA4ilEpY0dN2JSr+VHYn0EAABA67imDQAAAAAiRqUNAAAAACJWiO6RAIBwuC4LAIC4UGlDXYwqCQBhsD9FTFgfgfyheyQAAAAARIyWNgCF1M6osoxE2xj5hEemyaAlCWljnUPSEqu0mdkXJf2NpEWS/o+7jyf1Wo0UaSOKJdMiSTvTRgdo9eRtXc3DetrO55ClPGSaN2QaHpmGR6bJINfwYjmeytsxUysSqbSZ2SJJ35U0ImlG0vNmtsfdf5HE67Ujb2c3yTS8PGSaN2QaHpmGR6bh5SXTPH1P5SXTvCHX8Mg0HUm1tF0l6VV3/6UkmdkPJa2VxIfXPjINLxeZ5uxsUi4yzRkyDS8XmbLtpytk3oGeK/eZRopcw4sm05ztN1uSVKVthaRfVd2ekfTZhF4ruJBdpxqtJC2uWNFk2k4+aW1Eec20HWQah3Y+hyJmmlaX00DXI+Yi03rSul6zmzJNSxEzzfr64TaeK9VcY2jd5XsqjstTOj1uS6rSZjWW+VkPMNskaVP55n+Z2a8lnUyoPElaqgbltvtaf8J5//PRyuIaD10o0yMLlS9rVe810XK2m6mUv3XV7kvnMw+c6ZEARQr5vs95rla35XbW7Rqv8VFlm2m7ElsH29ynVpenk0yj3vYlLbX7Wi9foO+pTtbTPHxPJV7GGDJtZ10I/Xwhy1Bj25faP56SAq8HofNuwlnl5xi1rqWSTqb1+dTJ9BxJVdpmJF1cdbtf0hvVD3D3bZK2VW6b2QF3H06oPIlJsdwtZyrlJ9eMyrlgplL+1tWMy9dWpiGEfN8xPZeZXa2MMm1XbNvI/PK0m2ls72u+LMvXyXoae65SNmUseqZpqJNDW8dTDZ4vNxIsf6GOUWMtV1LztD0vacjMLjGzCyTdImlPQq/VLcg0PDINj0zDI9PwyDQ8Mg2PTJNBruGRaQoSaWlz9zNmtlnST1Qa+vMRdz+cxGt1CzINj0zDI9PwyDQ8Mg2PTMMj02SQa3hkmo7E5mlz9yclPdnCv0TRracNqZW7jUyl/OSaSTkLmmmm5Wsz0xBCvu+onivDTNsV2zZyTnnY9sPrYD2NPVcpX99RUj4yTUPNHLo418TKX7B9apTlMvdzrmkFAAAAAEQiqWvaAAAAAAABRFFpM7MvmtkRM3vVzMayLk89Znaxme03s1fM7LCZ/Xl5+RIz22dmR8u/L4ygrNFm2iDHLWb2H2Y2Vf5Zk3VZq8WWaZ7Wx6SZ2bSZvVxebw60+L+PmNlxMztUtaytDOs8V9TrdUidfA6BXj/YZ1njudn+A4stUym/30/VYsw1DbX2P0Xd/puVZCYdliuKPHO3H3X3TH9UumDxNUmXSrpA0kuSLsu6XHXK2ifp0+W/F0v6V0mXSfprSWPl5WOS7iPTtnLcIunurMuXl0zzsj6mlMW0pKVt/u8qSZ+WdKhqWVsZ1nmuaNfrmD6HQK8f7LOc97xs/+HLH12mC+Sai+041lxTeu/n7H+Kuv1nnUmHZYomz7ztR2NoabtK0qvu/kt3/62kH0pam3GZanL3WXd/sfz325JeUWkW+LWSdpQftkPSlzIp4O9EnWmDHGMWXaY5Wh+j5u4/k3Rq3uK2MqzzXEhJyM9yHrb/8KLLVMrt91O1KHPNUCG3/w5lvY+IJs+87UdjqLStkPSrqtszysEO0swGJf2ppJ9L6nX3Wam0AkjqybBoUo4ynZejJG02s4Plbk5xNEeXRJ1p5OtjGlzS02b2gpltCvB8oTOMdb0OLfTnEEKIz5LtP7yoM5Vy9f1ULfpcE1Rr/1P47X8BSWXSiSjzzMN+NIZKm9VYFvWQlmb2YUn/KOkv3P03WZenhlxkWiPHv5P0MUkrJc1Kuj+70p0j2kxzsD6m4XPu/mlJ/0PSV81sVdYFqhLzeh1azJ9DJ9j+w4s2Uyl330/Vos41YUntf/KcaYz75OjyzMt+NIZK24yki6tu90t6I6OyLMjMfk+lD/bv3f2fyouPmVlf+f4+ScezKl9Z9JnWytHdj7n7++7+gaSHVWpCj0WUmeZkfUycu79R/n1c0i51vu4EyzDy9TqoBD6HEEJ8lmz/4UWZqZTL76dq0eaatDr7n8Ju/81IMJNORJVnnvajMVTanpc0ZGaXmNkFkm6RtCfjMtVkZiZpu6RX3P07VXftkbSu/Pc6SbvTLts8UWdaL8fKBlJ2o6RD8/83Q9FlmqP1MVFm9iEzW1z5W9K16nzdCZZh5Ot1MAl9DiGE+CzZ/sOLLlMpt99P1aLMNWkN9j+F3P6bkXAmnYgmz9ztR7MeCcVLI7OsUWnEltckfTPr8jQo5+dVasI9KGmq/LNG0kWSJiQdLf9eEkFZo820QY4/kPRyefkeSX1ZlzXmTPO0Piacw6UqjT71kqTDrX42knaq1N3pPZXOAG5sN8M6zxX1eh3L5xCoDME+yxrPzfYf/j1ElekCueZmO44x1xTec839T1G3/xgy6bBsUeSZt/2olQsNAAAAAIhQDN0jAQAAAAB1UGkDAAAAgIhRaQMAAACAiFFpAwAAAICIUWkDAAAAgIhRaQMAAACAiFFpAwAAAICIUWkDAAAAgIj9/wHLGAEEu9mESQAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 1080x720 with 50 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "import matplotlib\n",
+ "matplotlib.rcParams['figure.figsize'] = [15, 10]\n",
+ "\n",
+ "problem_ids = list(student_scores.ProblemID.unique())\n",
+ "fig, axs = plt.subplots(5, 10)\n",
+ "for i in range(5):\n",
+ " for j in range(10):\n",
+ " problem_id = problem_ids[i * 10 + j]\n",
+ " attempts = student_scores[student_scores[PS2.ProblemID] == problem_id].Attempts\n",
+ " p75 = problem_attempt_75th[problem_id] + 1\n",
+ " axs[i, j].hist(attempts)\n",
+ " axs[i, j].vlines(p75, 0, 50)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "True 8014\n",
+ "False 2829\n",
+ "Name: Label, dtype: int64"
+ ]
+ },
+ "execution_count": 29,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "cutoffs = student_scores['ProblemID'].apply(lambda x: problem_attempt_75th[x])\n",
+ "student_scores['Label'] = np.logical_and(student_scores['Attempts'] <= cutoffs, student_scores['CorrectEventually'])\n",
+ "student_scores['Label'].value_counts()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(array([ 1., 1., 6., 16., 17., 23., 34., 41., 55., 52.]),\n",
+ " array([0.04347826, 0.13913043, 0.23478261, 0.33043478, 0.42608696,\n",
+ " 0.52173913, 0.6173913 , 0.71304348, 0.80869565, 0.90434783,\n",
+ " 1. ]),\n",
+ " <BarContainer object of 10 artists>)"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeAAAAD4CAYAAAA0JjXXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAANcElEQVR4nO3dbaykd1nH8d9lFwIKSmvPbjYt9aCpSGMs4IqNqAFqtRTj1gQMqLAhTTbGh2BiIisvNMY35Y0hRg3ZIGGNCjZS7MoqulmsaHjcaimtBVsRa8OmuxSUBxNNy+WLM+ha9nDu3TMz/57p55OczNz3zOxc+eecfPeemXOf6u4AAMv1daMHAIAnIgEGgAEEGAAGEGAAGECAAWCAXct8sksvvbTX19eX+ZQAMMwdd9zxme5eO9dtSw3w+vp6Tp48ucynBIBhqupfN7vNS9AAMIAAA8AAAgwAAwgwAAwgwAAwgAADwAACDAADCDAADCDAADDAUs+EBcDirB86NnqEr+lTN79s9AiPK46AAWAAAQaAAQQYAAYQYAAYQIABYAABBoABBBgABhBgABhAgAFgAAEGgAGcihKApXi8nyozWe7pMh0BA8AAAgwAAwgwAAww6T3gqvpUki8keTTJI929r6ouSfLHSdaTfCrJT3T35xYzJgCslvM5An5xdz+3u/fNtg8lOdHdVyY5MdsGACbYzkvQ+5McmV0/kuTGbU8DAE8QUwPcSf6qqu6oqoOzfXu6+1SSzC53L2JAAFhFU38P+IXd/emq2p3keFV9fOoTzIJ9MEmuuOKKCxgRAFbPpCPg7v707PJ0kncleUGSh6pqb5LMLk9v8tjD3b2vu/etra3NZ2oA2OG2DHBVfUNVPf0r15P8cJK7kxxNcmB2twNJblvUkACwaqa8BL0nybuq6iv3/6Pufk9VfSTJLVV1U5IHkrxicWMCwGrZMsDd/ckkV59j/8NJrl3EUACw6pwJCwAGEGAAGECAAWAAAQaAAQQYAAYQYAAYQIABYAABBoABBBgABhBgABhAgAFgAAEGgAEEGAAGEGAAGECAAWAAAQaAAQQYAAYQYAAYQIABYAABBoABBBgABhBgABhAgAFgAAEGgAEEGAAGEGAAGECAAWAAAQaAAXaNHgBgJ1g/dGz0CKyYyUfAVXVRVf1DVb17tn1JVR2vqvtmlxcvbkwAWC3n8xL065Lce9b2oSQnuvvKJCdm2wDABJMCXFWXJ3lZkrectXt/kiOz60eS3DjXyQBghU09An5Tkl9O8uWz9u3p7lNJMrvcfa4HVtXBqjpZVSfPnDmznVkBYGVsGeCq+tEkp7v7jgt5gu4+3N37unvf2trahfwTALBypnwK+oVJfqyqbkjylCTfWFV/kOShqtrb3aeqam+S04scFABWyZZHwN39K919eXevJ3llkvd2908nOZrkwOxuB5LctrApAWDFbOdEHDcnua6q7kty3WwbAJjgvE7E0d23J7l9dv3hJNfOfyQAWH1ORQkAAwgwAAwgwAAwgAADwAACDAADCDAADCDAADCAAAPAAAIMAAMIMAAMIMAAMIAAA8AAAgwAAwgwAAwgwAAwgAADwAACDAAD7Bo9AECSrB86NnoEWCpHwAAwgAADwAACDAADCDAADCDAADCAAAPAAAIMAAMIMAAMIMAAMIAAA8AAAgwAA2wZ4Kp6SlV9uKo+WlX3VNWvz/ZfUlXHq+q+2eXFix8XAFbDlCPg/0ryku6+Oslzk1xfVdckOZTkRHdfmeTEbBsAmGDLAPeGL842nzT76iT7kxyZ7T+S5MZFDAgAq2jSe8BVdVFV3ZnkdJLj3f2hJHu6+1SSzC53b/LYg1V1sqpOnjlzZk5jA8DONinA3f1odz83yeVJXlBV3zn1Cbr7cHfv6+59a2trFzgmAKyW8/oUdHf/e5Lbk1yf5KGq2psks8vT8x4OAFbVlE9Br1XVM2bXn5rkh5J8PMnRJAdmdzuQ5LYFzQgAK2fXhPvsTXKkqi7KRrBv6e53V9UHktxSVTcleSDJKxY4JwCslC0D3N13JXneOfY/nOTaRQwFAKvOmbAAYAABBoABBBgABhBgABhAgAFgAAEGgAEEGAAGEGAAGECAAWAAAQaAAQQYAAYQYAAYQIABYAABBoABBBgABhBgABhAgAFgAAEGgAEEGAAGEGAAGGDX6AGAxVs/dGz0CMBjOAIGgAEEGAAGEGAAGECAAWAAAQaAAQQYAAYQYAAYYMsAV9Uzq+qvq+reqrqnql43239JVR2vqvtmlxcvflwAWA1TjoAfSfJL3f2cJNck+bmquirJoSQnuvvKJCdm2wDABFsGuLtPdfffz65/Icm9SS5Lsj/JkdndjiS5cUEzAsDKOa/3gKtqPcnzknwoyZ7uPpVsRDrJ7k0ec7CqTlbVyTNnzmxzXABYDZMDXFVPS/LOJL/Y3Z+f+rjuPtzd+7p739ra2oXMCAArZ1KAq+pJ2YjvH3b3rbPdD1XV3tnte5OcXsyIALB6pnwKupL8XpJ7u/s3z7rpaJIDs+sHktw2//EAYDVN+XOEL0zy6iQfq6o7Z/vekOTmJLdU1U1JHkjyioVMCAAraMsAd/ffJalNbr52vuMAwBODM2EBwAACDAADCDAADCDAADCAAAPAAAIMAANM+T1gYAvrh46NHgHYYRwBA8AAAgwAAwgwAAzgPWAe97y/CqwiR8AAMIAAA8AAAgwAAwgwAAwgwAAwgAADwAACDAADCDAADCDAADCAAAPAAAIMAAMIMAAMIMAAMIAAA8AAAgwAAwgwAAwgwAAwgAADwABbBriq3lpVp6vq7rP2XVJVx6vqvtnlxYsdEwBWy5Qj4Lcluf4x+w4lOdHdVyY5MdsGACbaMsDd/b4kn33M7v1JjsyuH0ly43zHAoDVdqHvAe/p7lNJMrvcvdkdq+pgVZ2sqpNnzpy5wKcDgNWy8A9hdffh7t7X3fvW1tYW/XQAsCNcaIAfqqq9STK7PD2/kQBg9V1ogI8mOTC7fiDJbfMZBwCeGKb8GtLbk3wgybOr6sGquinJzUmuq6r7klw32wYAJtq11R26+1Wb3HTtnGcBgCcMZ8ICgAEEGAAGEGAAGECAAWAAAQaAAQQYAAYQYAAYQIABYAABBoABtjwTFqtt/dCx0SMAPCE5AgaAAQQYAAYQYAAYQIABYAABBoABBBgABhBgABhAgAFgAAEGgAEEGAAGEGAAGECAAWAAAQaAAQQYAAYQYAAYQIABYAABBoABBBgABtg1eoDtWD90bPQIAHBBtnUEXFXXV9Unqur+qjo0r6EAYNVdcICr6qIkv5PkpUmuSvKqqrpqXoMBwCrbzhHwC5Lc392f7O7/TvKOJPvnMxYArLbtvAd8WZJ/O2v7wSTf+9g7VdXBJAdnm1+sqk8kuTTJZ7bx3Hw1azp/1nS+rOf8WdM5qzfOfU2/ZbMbthPgOse+/qod3YeTHP5/D6w62d37tvHcPIY1nT9rOl/Wc/6s6fwtc0238xL0g0meedb25Uk+vb1xAOCJYTsB/kiSK6vqWVX15CSvTHJ0PmMBwGq74Jegu/uRqvr5JH+Z5KIkb+3ueyY+/PDWd+E8WdP5s6bzZT3nz5rO39LWtLq/6m1bAGDBnIoSAAYQYAAYYGEB3uo0lbXht2a331VVz1/ULKtiwpr+1Gwt76qq91fV1SPm3Emmnk61qr6nqh6tqpcvc76daMqaVtWLqurOqrqnqv5m2TPuNBN+9r+pqv6sqj46W9PXjphzp6iqt1bV6aq6e5Pbl9On7p77VzY+lPXPSb41yZOTfDTJVY+5zw1J/iIbv098TZIPLWKWVfmauKbfl+Ti2fWXWtPtr+lZ93tvkj9P8vLRcz+evyZ+nz4jyT8muWK2vXv03I/nr4lr+oYkb5xdX0vy2SRPHj374/UryQ8meX6Suze5fSl9WtQR8JTTVO5P8vu94YNJnlFVexc0zyrYck27+/3d/bnZ5gez8bvZbG7q6VR/Ick7k5xe5nA71JQ1/ckkt3b3A0nS3db1a5uypp3k6VVVSZ6WjQA/stwxd47ufl821mgzS+nTogJ8rtNUXnYB9+H/nO963ZSN/8GxuS3XtKouS/LjSd68xLl2sinfp9+e5OKqur2q7qiq1yxtup1pypr+dpLnZONkSB9L8rru/vJyxltJS+nTov4e8JTTVE46lSX/a/J6VdWLsxHg71/oRDvflDV9U5LXd/ejGwcXbGHKmu5K8t1Jrk3y1CQfqKoPdvc/LXq4HWrKmv5IkjuTvCTJtyU5XlV/292fX/Bsq2opfVpUgKecptKpLM/PpPWqqu9K8pYkL+3uh5c02041ZU33JXnHLL6XJrmhqh7p7j9dyoQ7z9Sf/c9095eSfKmq3pfk6iQCfG5T1vS1SW7ujTcw76+qf0nyHUk+vJwRV85S+rSol6CnnKbyaJLXzD5tdk2S/+juUwuaZxVsuaZVdUWSW5O82tHEJFuuaXc/q7vXu3s9yZ8k+Vnx/Zqm/OzfluQHqmpXVX19Nv6K2r1LnnMnmbKmD2TjFYVU1Z4kz07yyaVOuVqW0qeFHAH3JqeprKqfmd3+5mx8ovSGJPcn+c9s/A+OTUxc019N8s1Jfnd2xPZI+0spm5q4ppyHKWva3fdW1XuS3JXky0ne0t3n/HUQJn+f/kaSt1XVx7Lx8unru9ufKdxEVb09yYuSXFpVDyb5tSRPSpbbJ6eiBIABnAkLAAYQYAAYQIABYAABBoABBBgABhBgABhAgAFggP8BvjVhMXKKbbMAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ "<Figure size 576x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "matplotlib.rcParams['figure.figsize'] = [8, 4]\n",
+ "\n",
+ "# The percentage of struggling problems per student is well-distributed\n",
+ "plt.hist(student_scores.groupby(PS2.SubjectID)['Label'].mean())"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Identifying Late Assignments\n",
+ "\n",
+ "Below we confirm that the 5 assignments are well-spaced out, with a consistent ordering accross students.\n",
+ "\n",
+ "The latter 2 assignments are what is predicted in Task 1 of the data challenge.\n",
+ "\n",
+ "We divide the data by assignment, rather than by problem, since within a given assignment studens do problems in a variety of orders (see analysis at the end of this document)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "problem_times = runs.groupby([PS2.AssignmentID, PS2.ProblemID])['TimeInt'].median()\n",
+ "start_time = min(problem_times)\n",
+ "problem_times = (problem_times - start_time) / 10**9\n",
+ "\n",
+ "problem_successes = runs[runs[PS2.Score] == 1].groupby([PS2.AssignmentID, PS2.ProblemID])['TimeInt'].median()\n",
+ "problem_successes = (problem_successes - start_time) / 10**9"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>MedTime</th>\n",
+ " <th>MedSuccess</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>439.0</td>\n",
+ " <td>1</td>\n",
+ " <td>7960.5</td>\n",
+ " <td>8063.5</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>439.0</td>\n",
+ " <td>3</td>\n",
+ " <td>8269.5</td>\n",
+ " <td>8954.5</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>439.0</td>\n",
+ " <td>5</td>\n",
+ " <td>6310.0</td>\n",
+ " <td>6585.5</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>439.0</td>\n",
+ " <td>12</td>\n",
+ " <td>3891.5</td>\n",
+ " <td>7772.5</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>439.0</td>\n",
+ " <td>13</td>\n",
+ " <td>0.0</td>\n",
+ " <td>1005.5</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " AssignmentID ProblemID MedTime MedSuccess\n",
+ "0 439.0 1 7960.5 8063.5\n",
+ "1 439.0 3 8269.5 8954.5\n",
+ "2 439.0 5 6310.0 6585.5\n",
+ "3 439.0 12 3891.5 7772.5\n",
+ "4 439.0 13 0.0 1005.5"
+ ]
+ },
+ "execution_count": 32,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "problem_stats = problem_times.to_frame('MedTime').join(problem_successes.to_frame('MedSuccess')).reset_index()\n",
+ "problem_stats.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<matplotlib.collections.PathCollection at 0x2e512ddc160>"
+ ]
+ },
+ "execution_count": 33,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeAAAAEFCAYAAAA7XTSkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAUNUlEQVR4nO3dW4ycd3nH8e+zdlzY4Iq42URWnHhpa2gRKgkdRbSpUEkICgHh3ASBlsqqou4NoKAeUGAvKiqtxBXihlZaJWldMZCmHJoIRUBkQISKQ9YcCiHQRMEbrLjxEoNMulJx4qcX8xqvvbPr8Zzew3w/kjXz/md230evLP/8zjP//z8yE0mSNF5TZRcgSdIkMoAlSSqBASxJUgkMYEmSSmAAS5JUgu3jPNnll1+es7Oz4zylJEmlOXz48M8zc6bba2MN4NnZWZaXl8d5SkmSShMRK5u95kfQkiSVoKc74Ig4AvwKeBF4ITNbEbEL+DdgFjgCvCMzfzGaMiVJapaLuQN+Y2Zem5mt4vgu4FBm7gMOFceSJKkHg3wEvR84WDw/CNw2cDWSJE2IXgM4gS9FxOGImC/GrszMYwDF4xXdfjAi5iNiOSKWV1dXB69YkqQG6DWAb8jM1wFvAd4TEW/o9QSZuZSZrcxszcx0/Sa2JEnlardhdhampjqP7fbIT9nTl7Ay85ni8XhEfA64Hng2InZn5rGI2A0cH2GdkiSNRrsN8/OwttY5XlnpHAPMzY3stBe8A46ISyNi55nnwJuBHwIPAgeKtx0AHhhVkZIkjczCwtnwPWNtrTM+Qr3cAV8JfC4izrz/k5n5hYh4FLg/Iu4AngZuH12ZkiSNyNNPX9z4kFwwgDPzKeC1XcafA24aRVGSJI3NNdd0PnbuNj5CroQlSZpst956ceNDYgBLkibbQw9d3PiQGMCSpMlWUg/YAJYkTbZdu7qP2wOWJGlE2m04eXLj+I4dsLg40lMbwJKkybWwAKdObRzfuXOki3CAASxJmmSb9XlPnBj5qQ1gSdLk2qzPO+L+LxjAkqRJVtIcYDCAJUmTrKQ5wGAAS5ImWUlzgMEAliRNMnvAkiSVwB6wJEklsAcsSVIJ7AFLkjRm7TZMbRKD9oAlSRqBdhvm5+HFFze+Nj098nWgwQCWJE2ihQVYW9s4vm0bLC2NfB1oMIAlSZNosx7v6dNjCV8wgCVJk6ikPYDXM4AlSZOlxD2A1zOAJUmTpcQ9gNczgCVJk6XEPYDXM4AlSZOlxPWf1zOAJUmTpcT1n9czgCVJk6XE9Z/XM4AlSZOlxPWf1zOAJUmTxR6wJEklWFzszPldb8xzgMEAliRNosytj8fAAJYkTZZuC3GcOtUZH6OeAzgitkXEdyPi88Xxroh4OCKeKB4vG12ZkiQNSQ2/hHUn8Pi647uAQ5m5DzhUHEuSVG0V2IgBegzgiNgDvBW4e93wfuBg8fwgcNtQK5MkadgqshED9H4H/DHgA8DpdWNXZuYxgOLxiuGWJknSkFVkIwboIYAj4m3A8cw83M8JImI+IpYjYnl1dbWfXyFJ0nBUZCMG6O0O+Abg7RFxBLgPuDEiPgE8GxG7AYrH491+ODOXMrOVma2ZmZkhlS1JA2i3YXYWpqY6j+122RVpXCqyCAf0EMCZ+cHM3JOZs8A7gS9n5ruBB4EDxdsOAA+MrEpJGpZ2G+bnYWWlM/dzZaVzbAhPhsVFmJ4+d2x6euz9XxhsHvBHgJsj4gng5uJYkqptYQHW1s4dW1sb+xxQlWRuDpaWYO9eiOg8Li2Nvf8LEDnG1T9arVYuLy+P7XyStMHUVPdVjyLg9OmN49IAIuJwZra6veZKWJImS0XmgEoGsKTJUaE5oJIBLGlyVGgOqGQAS5ocFZoDKhnAkiaH/V9ViAEsaTLY/1XFGMCSJoP9X1WMASxpMtj/VcUYwJKa7cy6z5stOmT/VyXZXnYBkjQyZ9Z9Pn/pyTNKWgNYAu+AJTVZt3WfzyhxDWAJvAOW1GSb9X0j4MiRsZYinc87YEnNVaG9X6XzGcCSmuvWWy9uXBojA1hScz300MWNS2NkAEtqrs16wJuNS2NkAEtqLnvAqjADWFJz2QNWhRnAkprLHrAqzACW1Fz2gFVhBrCk5nL/X1WYASypmdz/VxVnAEtqJvf/VcUZwJKayf1/VXEGsKRmcg6wKs4AltRMzgFWxRnAkprJOcCqOANYUjM5B1gVZwBLaibnAKviDGBJzeMcYNWAASypeZwDrBowgCU1j3OAVQMXDOCIeElEfDsivh8Rj0XEh4vxXRHxcEQ8UTxeNvpyJWkT7TbMzsLUVOdPN/Z/VSG93AH/H3BjZr4WuBa4JSJeD9wFHMrMfcCh4liSxq/dhvl5WFmBTHjxxY3vmZ62/6tKuWAAZ8fzxeElxZ8E9gMHi/GDwG2jKFCSLmhhAdbWNo5v2wYRsHcvLC3Z/1WlbO/lTRGxDTgM/D7w8cz8VkRcmZnHADLzWERcscnPzgPzANf48Y+kUdis53v6dOePVEE9fQkrM1/MzGuBPcD1EfGaXk+QmUuZ2crM1szMTJ9lShW1vu84O9s51vi57rNq6KK+BZ2ZvwS+CtwCPBsRuwGKx+PDLk6qtPP7jisrnWNDePwWFzs93vXs+arievkW9ExEvLx4/lLgTcCPgQeBA8XbDgAPjKhGqZq69R3X1jrjGq+5uU6Pd+9ee76qjcjMrd8Q8Ud0vmS1jU5g35+Z/xARvwPcD1wDPA3cnplbTrJrtVq5vLw8lMKl0k1Nde58zxdh31ESABFxODNb3V674JewMvO/gOu6jD8H3DR4eVJNXXNN52PnbuOSdAGuhCX1y/1mJQ3AAJb65X6zkgZgAEv9cr9ZSQMwgKV+OfdU0gAMYKlf9oAlDcAAlvplD1jSAAxgqV/2gCUNwACW+mUPWNIADGCpX/aAJQ3AAJb6ZQ9Y0gAMYKlf9oAlDcAAlvq1a1f3cXvAknpgAEv9aLfh5MmN4zt2uAetpJ4YwFI/Fhbg1KmN4zt3ugetpJ4YwFI/NuvznthyS2xJ+g0DWOqHc4AlDcgAlvrhHGBJAzKApX44B1jSgAxgqR/OAZY0IANY6oc9YEkDMoClftgDljQgA1jqhz1gSQMygKV+2AOWNCADWOqHPWBJAzKApX7YA5Y0IANY6oc9YEkDMoClftgDljQgA1i6GO02zM5CZvfX7QFL6tH2sguQaqPdhvl5WFvr/vr0tHsBS+qZd8BSrxYWNg/fvXthacm9gCX1zDtgqVeb9Xcj4MiRsZYiqf4ueAccEVdHxFci4vGIeCwi7izGd0XEwxHxRPF42ejLlUrk3F9JQ9TLR9AvAH+TmX8IvB54T0S8GrgLOJSZ+4BDxbHUXM79lTREFwzgzDyWmd8pnv8KeBy4CtgPHCzedhC4bUQ1StXg3F9JQ3RRX8KKiFngOuBbwJWZeQw6IQ1cscnPzEfEckQsr66uDliuVCLn/koaop4DOCJeBnwGeH9mnuz15zJzKTNbmdmamZnpp0Z1c2Y+6tRU57HdLrui5rMHLGmIegrgiLiETvi2M/OzxfCzEbG7eH03cHw0JWqDM/NRV1Y6C0KsrHSODeHRWlzszPVdz7m/kvrUy7egA7gHeDwzP7rupQeBA8XzA8ADwy9PXXWbj7q21hnX6MzNdeb67t3bmXrk3F9JA4jcbEm9M2+I+DPgEeAHwOli+EN0+sD3A9cATwO3Z+aJrX5Xq9XK5eXlQWvW1FT3pRAj4PTpjeOSpFJExOHMbHV77YILcWTm14HY5OWbBilMfdq1C557buO4vUhJqg2XoqybdhtOdvkO3I4d9iIlqUYM4LpZWIBTpzaO79xpL1KSasQArpvN5pye2LL9LkmqGAO4bpyLKkmNYADXjesRS1IjGMB143rEktQIBnDduB6xJDWCAVwXZ9Z+3mzhFHvAklQrF1yIQxVwZu3n85efPMP1iCWpdrwDroNuaz+f4XrEklRL3gHXwWb93Qg4cmSspUiShsM74Dpw7q8kNY4BXAfO/ZWkxjGA68C5v5LUOAZwHTj3V5IaxwCug127uo/bA5ak2jKAq879fyWpkQzgqnP/X0lqJAO46tz/V5IayQCuOucAS1IjGcBV5xxgSWokA7jqnAMsSY1kAFedc4AlqZEM4KqzByxJjWQAV93iYmfO73rOAZak2jOA6yBz62NJUu0YwFXXbSGOU6c645Kk2jKAq84vYUlSIxnAVedGDJLUSAZwlbkRgyQ1lgFcZW7EIEmNdcEAjoh7I+J4RPxw3diuiHg4Ip4oHi8bbZkTyo0YJKmxerkD/hfglvPG7gIOZeY+4FBxrGFzEQ5JaqwLBnBmfg04/5ZrP3CweH4QuG24ZQlwIwZJarB+e8BXZuYxgOLxiuGVpN9wIwZJaqyRfwkrIuYjYjkilldXV0d9umZxDrAkNVa/AfxsROwGKB6Pb/bGzFzKzFZmtmZmZvo8XRftNszOwtRU57HdHt7vrgp7wJLUWP0G8IPAgeL5AeCB4ZTTo3Yb5udhZaWzLvLKSue4aSG8uAjT0+eOTU87B1iSGqCXaUifAr4BvCoijkbEHcBHgJsj4gng5uJ4fBYWYG3t3LG1teatjzw3B0tLsHcvRHQel5acAyxJDRA5xp11Wq1WLi8vD/6Lpqa67wgUAadPD/77JUkagog4nJmtbq/VcyUse6OSpJqrZwA7P1aSVHP1DGDnx0qSaq6eAez8WElSzdUzgO0BS5Jqrp4BbA9YklRz9Qxge8CSpJqrZwDbA5Yk1Vw9A9gesCSp5uoZwPaAJUk1V88AtgcsSaq5egawPWBJUs3VM4DtAUuSaq6eAWwPWJJUc/UMYHvAkqSaq2cA2wOWJNVcPQPYHrAkqebqGcD2gCVJNVfPALYHLEmquXoGsD1gSVLN1TOA7QFLkmqungFsD1iSVHP1DGB7wJKkmqtnANsDliTVXD0D2B6wJKnm6hnA9oAlSTVXzwC2ByxJqrl6BrA9YElSzdUzgHft6j5uD1iSVBP1C+B2G06e3Di+YwcsLo6/HkmS+lC/AF5YgFOnNo7v3Alzc+OvR5KkPtQvgDfr8544Md46JEkawEABHBG3RMRPIuLJiLhrWEVtyTnAkqQG6DuAI2Ib8HHgLcCrgXdFxKuHVdimnAMsSWqAQe6ArweezMynMvPXwH3A/uGUtQXnAEuSGmCQAL4K+Nm646PF2DkiYj4iliNieXV1dYDTFZwDLElqgEECOLqM5YaBzKXMbGVma2ZmZoDTFewBS5IaYJAAPgpcve54D/DMYOX0YHERpqfPHZuedg6wJKlWBgngR4F9EfGKiNgBvBN4cDhlbWFuDpaWYO9eiOg8Li05B1iSVCvb+/3BzHwhIt4LfBHYBtybmY8NrbKtzM0ZuJKkWus7gAEy8yHArx9LknSR6rcSliRJDWAAS5JUAgNYkqQSGMCSJJUgMjesnTG6k0WsAitD/JWXAz8f4u+rM6/FWV6Lc3k9zvJanOW1OGuU12JvZnZdhWqsATxsEbGcma2y66gCr8VZXotzeT3O8lqc5bU4q6xr4UfQkiSVwACWJKkEdQ/gpbILqBCvxVlei3N5Pc7yWpzltTirlGtR6x6wJEl1Vfc7YEmSaskAliSpBLUN4Ii4JSJ+EhFPRsRdZddTloi4NyKOR8QPy66lbBFxdUR8JSIej4jHIuLOsmsqS0S8JCK+HRHfL67Fh8uuqWwRsS0ivhsRny+7ljJFxJGI+EFEfC8ilsuup0wR8fKI+HRE/Lj4d+NPxnr+OvaAI2Ib8N/AzcBROnsTvyszf1RqYSWIiDcAzwP/mpmvKbueMkXEbmB3Zn4nInYCh4HbJvTvRQCXZubzEXEJ8HXgzsz8ZsmllSYi/hpoAb+dmW8ru56yRMQRoJWZE78IR0QcBB7JzLuLfe2nM/OX4zp/Xe+ArweezMynMvPXwH3A/pJrKkVmfg04UXYdVZCZxzLzO8XzXwGPA1eVW1U5suP54vCS4k/9/rc9JBGxB3grcHfZtagaIuK3gTcA9wBk5q/HGb5Q3wC+CvjZuuOjTOg/tOouImaB64BvlVxKaYqPXL8HHAcezsyJvRbAx4APAKdLrqMKEvhSRByOiPmyiynR7wKrwD8XrYm7I+LScRZQ1wCOLmMT+797nSsiXgZ8Bnh/Zp4su56yZOaLmXktsAe4PiImskUREW8Djmfm4bJrqYgbMvN1wFuA9xRtrEm0HXgd8E+ZeR3wv8BYv09U1wA+Cly97ngP8ExJtahCin7nZ4B2Zn627HqqoPhY7avALeVWUpobgLcXvc/7gBsj4hPlllSezHymeDwOfI5OS28SHQWOrvtk6NN0Anls6hrAjwL7IuIVReP8ncCDJdekkhVfPLoHeDwzP1p2PWWKiJmIeHnx/KXAm4Afl1pUSTLzg5m5JzNn6fxb8eXMfHfJZZUiIi4tvqBI8XHrm4GJnEGRmf8D/CwiXlUM3QSM9Qub28d5smHJzBci4r3AF4FtwL2Z+VjJZZUiIj4F/DlweUQcBf4+M+8pt6rS3AD8BfCDovcJ8KHMfKi8kkqzGzhYzBiYAu7PzImefiMArgQ+1/m/KtuBT2bmF8otqVTvA9rFjdxTwF+O8+S1nIYkSVLd1fUjaEmSas0AliSpBAawJEklMIAlSSqBASxJUhcXu9lNRLwjIn5UbIDyyQu+329BS5K00cVsdhMR+4D7gRsz8xcRcUWx2MmmvAOWJKmLbpvdRMTvRcQXirW0H4mIPyhe+ivg45n5i+JntwxfMIAlSboYS8D7MvOPgb8F/rEYfyXwyoj4z4j4ZkRccOnXWq6EJUnSuBUbvfwp8O/FamIAv1U8bgf20VmZcA/wSES8ZqstDg1gSZJ6MwX8sthl7HxHgW9m5ingpxHxEzqB/OhWv0ySJF1Asb3pTyPiduhsABMRry1e/g/gjcX45XQ+kn5qq99nAEuS1EWx2c03gFdFxNGIuAOYA+6IiO8DjwH7i7d/EXguIn4EfAX4u8x8bsvf7zQkSZLGzztgSZJKYABLklQCA1iSpBIYwJIklcAAliSpBAawJEklMIAlSSrB/wPnXfQVYcrXOAAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ "<Figure size 576x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "y = range(0, len(problem_stats.index))\n",
+ "problem_stats.sort_values('MedTime', inplace=True)\n",
+ "plt.scatter(problem_stats['MedTime'], y, c='red')\n",
+ "# plt.scatter(problem_stats['MedSuccess'], y, c='blue')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "AssignmentID\n",
+ "439.0 1.551041e+18\n",
+ "487.0 1.551933e+18\n",
+ "492.0 1.553454e+18\n",
+ "494.0 1.555024e+18\n",
+ "502.0 1.557064e+18\n",
+ "dtype: float64"
+ ]
+ },
+ "execution_count": 34,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Assignments are, thankfully, already in order\n",
+ "assignment_stats = runs.groupby(PS2.AssignmentID).apply(lambda x: np.median(x['TimeInt']))\n",
+ "assignment_stats"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[439.0, 487.0, 492.0, 494.0, 502.0]"
+ ]
+ },
+ "execution_count": 35,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "valid_assignments = list(assignment_stats.sort_values().index)\n",
+ "if (semester == 'F19'):\n",
+ " valid_assignments.remove(NEW_F19_ASSIGNMENT)\n",
+ "valid_assignments"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[494.0, 502.0]"
+ ]
+ },
+ "execution_count": 36,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "late_assignments = valid_assignments[-2:]\n",
+ "late_assignments"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[439.0, 487.0, 492.0]"
+ ]
+ },
+ "execution_count": 37,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "early_assignments = valid_assignments[:-2]\n",
+ "early_assignments"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Attempts</th>\n",
+ " <th>CorrectEventually</th>\n",
+ " <th>Label</th>\n",
+ " <th>IsLateProblem</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>3</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>5</td>\n",
+ " <td>3</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>12</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>13</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>5</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>232</td>\n",
+ " <td>11</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>6</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>233</td>\n",
+ " <td>7</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>7</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>234</td>\n",
+ " <td>7</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>8</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>235</td>\n",
+ " <td>3</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>9</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>236</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>10</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>487.0</td>\n",
+ " <td>17</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>11</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>487.0</td>\n",
+ " <td>20</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>12</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>487.0</td>\n",
+ " <td>21</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>13</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>487.0</td>\n",
+ " <td>22</td>\n",
+ " <td>6</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>14</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>487.0</td>\n",
+ " <td>24</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>15</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>487.0</td>\n",
+ " <td>25</td>\n",
+ " <td>9</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>16</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>487.0</td>\n",
+ " <td>28</td>\n",
+ " <td>4</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>17</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>487.0</td>\n",
+ " <td>100</td>\n",
+ " <td>18</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>18</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>487.0</td>\n",
+ " <td>101</td>\n",
+ " <td>22</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>19</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>487.0</td>\n",
+ " <td>102</td>\n",
+ " <td>45</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>20</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>492.0</td>\n",
+ " <td>31</td>\n",
+ " <td>3</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>21</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>492.0</td>\n",
+ " <td>32</td>\n",
+ " <td>37</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>22</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>492.0</td>\n",
+ " <td>33</td>\n",
+ " <td>7</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>23</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>492.0</td>\n",
+ " <td>34</td>\n",
+ " <td>30</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>24</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>492.0</td>\n",
+ " <td>36</td>\n",
+ " <td>5</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>25</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>492.0</td>\n",
+ " <td>37</td>\n",
+ " <td>28</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>26</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>492.0</td>\n",
+ " <td>38</td>\n",
+ " <td>11</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>27</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>492.0</td>\n",
+ " <td>39</td>\n",
+ " <td>13</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>28</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>492.0</td>\n",
+ " <td>40</td>\n",
+ " <td>19</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>29</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>492.0</td>\n",
+ " <td>128</td>\n",
+ " <td>16</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>30</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>41</td>\n",
+ " <td>4</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>31</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>43</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>32</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>44</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>33</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>46</td>\n",
+ " <td>3</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>34</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>49</td>\n",
+ " <td>5</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>35</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>67</td>\n",
+ " <td>5</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>36</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>104</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>37</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>106</td>\n",
+ " <td>8</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>38</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>107</td>\n",
+ " <td>5</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>39</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>108</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>40</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>502.0</td>\n",
+ " <td>45</td>\n",
+ " <td>9</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>41</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>502.0</td>\n",
+ " <td>48</td>\n",
+ " <td>3</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>42</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>502.0</td>\n",
+ " <td>51</td>\n",
+ " <td>10</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>43</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>502.0</td>\n",
+ " <td>56</td>\n",
+ " <td>5</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>44</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>502.0</td>\n",
+ " <td>57</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>45</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>502.0</td>\n",
+ " <td>64</td>\n",
+ " <td>10</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>46</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>502.0</td>\n",
+ " <td>70</td>\n",
+ " <td>9</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>47</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>502.0</td>\n",
+ " <td>71</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>48</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>502.0</td>\n",
+ " <td>112</td>\n",
+ " <td>9</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>49</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>502.0</td>\n",
+ " <td>118</td>\n",
+ " <td>5</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Attempts \\\n",
+ "0 04c32d4d95425f73b3a1d6502aed4d48 439.0 1 1 \n",
+ "1 04c32d4d95425f73b3a1d6502aed4d48 439.0 3 2 \n",
+ "2 04c32d4d95425f73b3a1d6502aed4d48 439.0 5 3 \n",
+ "3 04c32d4d95425f73b3a1d6502aed4d48 439.0 12 1 \n",
+ "4 04c32d4d95425f73b3a1d6502aed4d48 439.0 13 2 \n",
+ "5 04c32d4d95425f73b3a1d6502aed4d48 439.0 232 11 \n",
+ "6 04c32d4d95425f73b3a1d6502aed4d48 439.0 233 7 \n",
+ "7 04c32d4d95425f73b3a1d6502aed4d48 439.0 234 7 \n",
+ "8 04c32d4d95425f73b3a1d6502aed4d48 439.0 235 3 \n",
+ "9 04c32d4d95425f73b3a1d6502aed4d48 439.0 236 2 \n",
+ "10 04c32d4d95425f73b3a1d6502aed4d48 487.0 17 1 \n",
+ "11 04c32d4d95425f73b3a1d6502aed4d48 487.0 20 2 \n",
+ "12 04c32d4d95425f73b3a1d6502aed4d48 487.0 21 1 \n",
+ "13 04c32d4d95425f73b3a1d6502aed4d48 487.0 22 6 \n",
+ "14 04c32d4d95425f73b3a1d6502aed4d48 487.0 24 1 \n",
+ "15 04c32d4d95425f73b3a1d6502aed4d48 487.0 25 9 \n",
+ "16 04c32d4d95425f73b3a1d6502aed4d48 487.0 28 4 \n",
+ "17 04c32d4d95425f73b3a1d6502aed4d48 487.0 100 18 \n",
+ "18 04c32d4d95425f73b3a1d6502aed4d48 487.0 101 22 \n",
+ "19 04c32d4d95425f73b3a1d6502aed4d48 487.0 102 45 \n",
+ "20 04c32d4d95425f73b3a1d6502aed4d48 492.0 31 3 \n",
+ "21 04c32d4d95425f73b3a1d6502aed4d48 492.0 32 37 \n",
+ "22 04c32d4d95425f73b3a1d6502aed4d48 492.0 33 7 \n",
+ "23 04c32d4d95425f73b3a1d6502aed4d48 492.0 34 30 \n",
+ "24 04c32d4d95425f73b3a1d6502aed4d48 492.0 36 5 \n",
+ "25 04c32d4d95425f73b3a1d6502aed4d48 492.0 37 28 \n",
+ "26 04c32d4d95425f73b3a1d6502aed4d48 492.0 38 11 \n",
+ "27 04c32d4d95425f73b3a1d6502aed4d48 492.0 39 13 \n",
+ "28 04c32d4d95425f73b3a1d6502aed4d48 492.0 40 19 \n",
+ "29 04c32d4d95425f73b3a1d6502aed4d48 492.0 128 16 \n",
+ "30 04c32d4d95425f73b3a1d6502aed4d48 494.0 41 4 \n",
+ "31 04c32d4d95425f73b3a1d6502aed4d48 494.0 43 2 \n",
+ "32 04c32d4d95425f73b3a1d6502aed4d48 494.0 44 2 \n",
+ "33 04c32d4d95425f73b3a1d6502aed4d48 494.0 46 3 \n",
+ "34 04c32d4d95425f73b3a1d6502aed4d48 494.0 49 5 \n",
+ "35 04c32d4d95425f73b3a1d6502aed4d48 494.0 67 5 \n",
+ "36 04c32d4d95425f73b3a1d6502aed4d48 494.0 104 1 \n",
+ "37 04c32d4d95425f73b3a1d6502aed4d48 494.0 106 8 \n",
+ "38 04c32d4d95425f73b3a1d6502aed4d48 494.0 107 5 \n",
+ "39 04c32d4d95425f73b3a1d6502aed4d48 494.0 108 1 \n",
+ "40 04c32d4d95425f73b3a1d6502aed4d48 502.0 45 9 \n",
+ "41 04c32d4d95425f73b3a1d6502aed4d48 502.0 48 3 \n",
+ "42 04c32d4d95425f73b3a1d6502aed4d48 502.0 51 10 \n",
+ "43 04c32d4d95425f73b3a1d6502aed4d48 502.0 56 5 \n",
+ "44 04c32d4d95425f73b3a1d6502aed4d48 502.0 57 2 \n",
+ "45 04c32d4d95425f73b3a1d6502aed4d48 502.0 64 10 \n",
+ "46 04c32d4d95425f73b3a1d6502aed4d48 502.0 70 9 \n",
+ "47 04c32d4d95425f73b3a1d6502aed4d48 502.0 71 1 \n",
+ "48 04c32d4d95425f73b3a1d6502aed4d48 502.0 112 9 \n",
+ "49 04c32d4d95425f73b3a1d6502aed4d48 502.0 118 5 \n",
+ "\n",
+ " CorrectEventually Label IsLateProblem \n",
+ "0 True True False \n",
+ "1 True True False \n",
+ "2 True True False \n",
+ "3 True True False \n",
+ "4 True True False \n",
+ "5 True False False \n",
+ "6 True False False \n",
+ "7 True False False \n",
+ "8 True True False \n",
+ "9 True True False \n",
+ "10 True True False \n",
+ "11 True True False \n",
+ "12 True True False \n",
+ "13 True False False \n",
+ "14 True True False \n",
+ "15 True False False \n",
+ "16 True True False \n",
+ "17 True False False \n",
+ "18 True False False \n",
+ "19 True False False \n",
+ "20 True True False \n",
+ "21 True False False \n",
+ "22 True False False \n",
+ "23 True False False \n",
+ "24 True True False \n",
+ "25 True False False \n",
+ "26 True False False \n",
+ "27 True False False \n",
+ "28 True False False \n",
+ "29 True False False \n",
+ "30 True False True \n",
+ "31 True True True \n",
+ "32 True True True \n",
+ "33 True True True \n",
+ "34 True False True \n",
+ "35 True True True \n",
+ "36 True True True \n",
+ "37 True False True \n",
+ "38 True True True \n",
+ "39 True True True \n",
+ "40 True False True \n",
+ "41 True True True \n",
+ "42 True False True \n",
+ "43 True False True \n",
+ "44 True True True \n",
+ "45 True False True \n",
+ "46 True False True \n",
+ "47 True True True \n",
+ "48 True False True \n",
+ "49 True False True "
+ ]
+ },
+ "execution_count": 38,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "student_scores['IsLateProblem'] = student_scores[PS2.AssignmentID].isin(late_assignments)\n",
+ "student_scores.sort_values([PS2.SubjectID, PS2.AssignmentID, 'IsLateProblem', PS2.ProblemID], inplace=True)\n",
+ "# Remove attempts not in a valid assignment (for F19)\n",
+ "student_scores = student_scores[student_scores[PS2.AssignmentID].isin(valid_assignments)]\n",
+ "student_scores.head(50)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Train/Test Split\n",
+ "\n",
+ "Here we split out data into training/test datasets, as well as eary problems (used to extract features input into the model) and late problems (where struggle will be predicted)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Split by SubjectID"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 39,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1.0"
+ ]
+ },
+ "execution_count": 39,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Valid subjects must have completed at least one early and one late problem\n",
+ "ealry_late_subject_ids = student_scores.groupby(PS2.SubjectID)['IsLateProblem'].apply(lambda x: np.mean(x) > 0 and np.mean(x) < 1)\n",
+ "# The vast majority of students have\n",
+ "np.mean(ealry_late_subject_ids)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 40,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Instersect these subjectIDs with the ones who completed the final exam\n",
+ "valid_subject_ids = subject_ids.intersection(set(ealry_late_subject_ids.index[ealry_late_subject_ids]))\n",
+ "valid_subject_ids = list(valid_subject_ids)\n",
+ "valid_subject_ids.sort()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "246\n",
+ "246\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(len(subject_ids))\n",
+ "print(len(valid_subject_ids))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_ids, test_ids = train_test_split(list(valid_subject_ids), test_size=0.25, random_state=1234)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "FileNotFoundError",
+ "evalue": "[Errno 2] No such file or directory: 'data\\\\Release\\\\S19\\\\split.csv'",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
+ "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_12600/3474547466.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 9\u001b[0m \u001b[1;31m# If saving, uncomment the top line\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 10\u001b[0m \u001b[1;31m# ids_df.to_csv(os.path.join(path, 'split.csv'), index=False)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mcached_df\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'split.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 13\u001b[0m \u001b[1;32massert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mids_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mequals\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcached_df\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\util\\_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m 309\u001b[0m \u001b[0mstacklevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 310\u001b[0m )\n\u001b[1;32m--> 311\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 312\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 313\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[0;32m 584\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 585\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 586\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 587\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 588\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m 480\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 481\u001b[0m \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 482\u001b[1;33m \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 483\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 484\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m 809\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 810\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 811\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 812\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 813\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m 1038\u001b[0m )\n\u001b[0;32m 1039\u001b[0m \u001b[1;31m# error: Too many arguments for \"ParserBase\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1040\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mmapping\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# type: ignore[call-arg]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1041\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1042\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\c_parser_wrapper.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m 49\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 50\u001b[0m \u001b[1;31m# open handles\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_open_handles\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 52\u001b[0m \u001b[1;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandles\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 53\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\base_parser.py\u001b[0m in \u001b[0;36m_open_handles\u001b[1;34m(self, src, kwds)\u001b[0m\n\u001b[0;32m 220\u001b[0m \u001b[0mLet\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreaders\u001b[0m \u001b[0mopen\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0mafter\u001b[0m \u001b[0mthey\u001b[0m \u001b[0mare\u001b[0m \u001b[0mdone\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mtheir\u001b[0m \u001b[0mpotential\u001b[0m \u001b[0mraises\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 221\u001b[0m \"\"\"\n\u001b[1;32m--> 222\u001b[1;33m self.handles = get_handle(\n\u001b[0m\u001b[0;32m 223\u001b[0m \u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 224\u001b[0m \u001b[1;34m\"r\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\common.py\u001b[0m in \u001b[0;36mget_handle\u001b[1;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[0;32m 700\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;34m\"b\"\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 701\u001b[0m \u001b[1;31m# Encoding\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 702\u001b[1;33m handle = open(\n\u001b[0m\u001b[0;32m 703\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 704\u001b[0m \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data\\\\Release\\\\S19\\\\split.csv'"
+ ]
+ }
+ ],
+ "source": [
+ "# Check the train/test split dataframe against the last saved run\n",
+ "ids_df = pd.DataFrame({PS2.SubjectID: valid_subject_ids})\n",
+ "ids_df['IsTrain'] = ids_df[PS2.SubjectID].isin(train_ids)\n",
+ "ids_df\n",
+ "\n",
+ "path = os.path.join('data', 'Release', semester)\n",
+ "os.makedirs(path, exist_ok=True)\n",
+ "\n",
+ "# If saving, uncomment the top line\n",
+ "# ids_df.to_csv(os.path.join(path, 'split.csv'), index=False)\n",
+ "cached_df = pd.read_csv(os.path.join(path, 'split.csv'))\n",
+ "\n",
+ "assert(ids_df.equals(cached_df))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "184\n",
+ "62\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(len(train_ids))\n",
+ "print(len(test_ids))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "student_scores_train = student_scores[student_scores[PS2.SubjectID].isin(train_ids)]\n",
+ "student_scores_test = student_scores[student_scores[PS2.SubjectID].isin(test_ids)]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 46,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "10843\n",
+ "7933\n",
+ "2910\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(student_scores.shape[0])\n",
+ "print(student_scores_train.shape[0])\n",
+ "print(student_scores_test.shape[0])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Task 1: Split Into early/late datasets\n",
+ "In Task 1, we need an early set of problems to use to extract features for the model, and a late set of problems where we're actually predicting student outcomes."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 47,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "early_train = student_scores_train[student_scores_train['IsLateProblem'] == False].drop(['IsLateProblem'], axis=1)\n",
+ "early_test = student_scores_test[student_scores_test['IsLateProblem'] == False].drop(['IsLateProblem'], axis=1)\n",
+ "late_train = student_scores_train[student_scores_train['IsLateProblem']].drop(['IsLateProblem', 'Attempts', 'CorrectEventually'], axis=1)\n",
+ "late_test = student_scores_test[student_scores_test['IsLateProblem']].drop(['IsLateProblem', 'Attempts', 'CorrectEventually'], axis=1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 48,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4867, 6)\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Attempts</th>\n",
+ " <th>CorrectEventually</th>\n",
+ " <th>Label</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>3</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>5</td>\n",
+ " <td>3</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>12</td>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>439.0</td>\n",
+ " <td>13</td>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Attempts \\\n",
+ "0 04c32d4d95425f73b3a1d6502aed4d48 439.0 1 1 \n",
+ "1 04c32d4d95425f73b3a1d6502aed4d48 439.0 3 2 \n",
+ "2 04c32d4d95425f73b3a1d6502aed4d48 439.0 5 3 \n",
+ "3 04c32d4d95425f73b3a1d6502aed4d48 439.0 12 1 \n",
+ "4 04c32d4d95425f73b3a1d6502aed4d48 439.0 13 2 \n",
+ "\n",
+ " CorrectEventually Label \n",
+ "0 True True \n",
+ "1 True True \n",
+ "2 True True \n",
+ "3 True True \n",
+ "4 True True "
+ ]
+ },
+ "execution_count": 48,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "print(early_train.shape)\n",
+ "early_train.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "print(early_test.shape)\n",
+ "early_test.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 49,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(3066, 4)\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Label</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>30</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>41</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>31</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>43</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>32</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>44</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>33</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>46</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>34</th>\n",
+ " <td>04c32d4d95425f73b3a1d6502aed4d48</td>\n",
+ " <td>494.0</td>\n",
+ " <td>49</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Label\n",
+ "30 04c32d4d95425f73b3a1d6502aed4d48 494.0 41 False\n",
+ "31 04c32d4d95425f73b3a1d6502aed4d48 494.0 43 True\n",
+ "32 04c32d4d95425f73b3a1d6502aed4d48 494.0 44 True\n",
+ "33 04c32d4d95425f73b3a1d6502aed4d48 494.0 46 True\n",
+ "34 04c32d4d95425f73b3a1d6502aed4d48 494.0 49 False"
+ ]
+ },
+ "execution_count": 49,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "print(late_train.shape)\n",
+ "late_train.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(1135, 4)\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>SubjectID</th>\n",
+ " <th>AssignmentID</th>\n",
+ " <th>ProblemID</th>\n",
+ " <th>Label</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>250</th>\n",
+ " <td>08fcb5ead4e963a6f0bbdbc971f4a3ee</td>\n",
+ " <td>494.0</td>\n",
+ " <td>41</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>251</th>\n",
+ " <td>08fcb5ead4e963a6f0bbdbc971f4a3ee</td>\n",
+ " <td>494.0</td>\n",
+ " <td>43</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>252</th>\n",
+ " <td>08fcb5ead4e963a6f0bbdbc971f4a3ee</td>\n",
+ " <td>494.0</td>\n",
+ " <td>44</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>253</th>\n",
+ " <td>08fcb5ead4e963a6f0bbdbc971f4a3ee</td>\n",
+ " <td>494.0</td>\n",
+ " <td>46</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>254</th>\n",
+ " <td>08fcb5ead4e963a6f0bbdbc971f4a3ee</td>\n",
+ " <td>494.0</td>\n",
+ " <td>49</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " SubjectID AssignmentID ProblemID Label\n",
+ "250 08fcb5ead4e963a6f0bbdbc971f4a3ee 494.0 41 True\n",
+ "251 08fcb5ead4e963a6f0bbdbc971f4a3ee 494.0 43 True\n",
+ "252 08fcb5ead4e963a6f0bbdbc971f4a3ee 494.0 44 True\n",
+ "253 08fcb5ead4e963a6f0bbdbc971f4a3ee 494.0 46 False\n",
+ "254 08fcb5ead4e963a6f0bbdbc971f4a3ee 494.0 49 True"
+ ]
+ },
+ "execution_count": 50,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "print(late_test.shape)\n",
+ "late_test.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 51,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "base_path = os.path.join('data', 'Release', semester)\n",
+ "os.makedirs(os.path.join(base_path, 'Train'), exist_ok=True)\n",
+ "os.makedirs(os.path.join(base_path, 'Test'), exist_ok=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 52,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "early_train.to_csv(os.path.join(base_path, 'Train', 'early.csv'), index=False)\n",
+ "late_train.to_csv(os.path.join(base_path, 'Train', 'late.csv'), index=False)\n",
+ "early_test.to_csv(os.path.join(base_path, 'Test', 'early.csv'), index=False)\n",
+ "late_test.drop('Label', axis=1).to_csv(os.path.join(base_path, 'Test', 'late.csv'), index=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data.save_subset(os.path.join(base_path, 'Train', 'Data'), lambda df: df[df[PS2.SubjectID].isin(train_ids) & df[PS2.AssignmentID].isin(valid_assignments)])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data.save_subset(os.path.join(base_path, 'Test', 'Data'), lambda df: df[df[PS2.SubjectID].isin(test_ids) & df[PS2.AssignmentID].isin(early_assignments)], False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Task 2: Predicting Student Grades\n",
+ "For Task 2, we are predicting students' Final Exam grades. We just add the appropriate LinkTable _without_ the actual grades, just leaving the SubjectIDs to predict for."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "student_table_dir = os.path.join(base_path, 'Test', 'Data', 'LinkTables')\n",
+ "os.makedirs(student_table_dir, exist_ok=True)\n",
+ "student_table_test = student_table[student_table[PS2.SubjectID].isin(test_ids)]\n",
+ "student_table_test.drop('X-Grade', axis=1).to_csv(os.path.join(student_table_dir, 'Subject.csv'), index=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Save the solution\n",
+ "The solution is, of course, not released, but used to judge submissions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sol_path = os.path.join('data', 'Solution', semester, 'task1', 'ref')\n",
+ "os.makedirs(sol_path, exist_ok=True)\n",
+ "late_test.to_csv(os.path.join(sol_path, 'truth.csv'), index=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sol_path = os.path.join('data', 'Solution', semester, 'task2', 'ref')\n",
+ "os.makedirs(sol_path, exist_ok=True)\n",
+ "student_table_test.to_csv(os.path.join(sol_path, 'truth.csv'), index=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Unused code for identifying the late problems\n",
+ "\n",
+ "The code below was used to investigate whether we could predict the last 3 problems of every assignment using the first 7. However, students appear to do the assignment in a variety of orders, making this difficult. Additionally, this would leak future data (e.g. the first 7 problems on Assignment 2 could be used to predict the last 3 problems on Assignment 1)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_start_end_order(rows):\n",
+ " return pd.Series({\n",
+ " 'StartEventOrder': min(rows[PS2.Order])\n",
+ " # 'EndEventOrder': max(rows[PS2.Order])\n",
+ " })\n",
+ "\n",
+ "start_orders = main_table_filtered.groupby([PS2.SubjectID, PS2.AssignmentID, PS2.ProblemID]).apply(get_start_end_order)\n",
+ "start_orders"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Unfortunately, the last 3 problems aren't always easy to pick apart\n",
+ "print(problem_stats[PS2.AssignmentID].unique())\n",
+ "assignment1 = problem_stats[problem_stats[PS2.AssignmentID] == 439]\n",
+ "y = range(0, 10)\n",
+ "plt.scatter(assignment1['MedTime'], y, c='red')\n",
+ "plt.scatter(assignment1['MedSuccess'], y, c='blue')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "late_problems = problem_stats.groupby(PS2.AssignmentID).apply(lambda rows: list(rows[PS2.ProblemID][rows['MedTime'].argsort()][-3:]))\n",
+ "late_problems"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "late_problem_ids_old = [st for row in late_problems for st in row]\n",
+ "late_problem_ids_old"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "runs_time = runs.copy()\n",
+ "runs_time['IsLateProblemInAssignment'] = runs_time[PS2.ProblemID].isin(late_problem_ids_old)\n",
+ "runs_time.head(15)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "first_late_attempts = runs_time[runs_time['IsLateProblemInAssignment']].groupby([PS2.AssignmentID, PS2.SubjectID])['TimeInt'].min()\n",
+ "first_late_attempts"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "runs_late = runs_time.merge(first_late_attempts.to_frame('FirstLateAttempt'), on=[PS2.AssignmentID, PS2.SubjectID], how='left')\n",
+ "np.mean(runs_late['FirstLateAttempt'].isna())\n",
+ "# ~2% of student-assignments did not have a late attempt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "runs_late['IsLateAttempt'] = runs_late['TimeInt'] >= runs_late['FirstLateAttempt']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "np.sum(~runs_late['IsLateAttempt'] & runs_late['IsLateProblemInAssignment'])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 30% of attempts at early problems occurred after the first attempt at a late problem\n",
+ "np.sum(runs_late['IsLateAttempt'] & ~runs_late['IsLateProblemInAssignment']) / np.sum(~runs_late['IsLateProblemInAssignment'])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "np.random.seed(1234)\n",
+ "subjects_sample = np.random.choice(runs_time[PS2.SubjectID].unique(), 20)\n",
+ "assignment_sample = runs_time[(runs_time[PS2.AssignmentID] == 439) & runs_time[PS2.SubjectID].isin(subjects_sample)]\n",
+ "assignment_sample.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "all_subject_ids = assignment_sample[PS2.SubjectID]\n",
+ "distinct_subject_ids = all_subject_ids.unique()\n",
+ "subject_indices = [sorted(distinct_subject_ids).index(x) for x in all_subject_ids]\n",
+ "colors = [viridis(float(i)) for i in assignment_sample['IsLateProblemInAssignment']]\n",
+ "subject_times_norm = assignment_sample.groupby('SubjectID')['TimeInt'].transform(lambda x: (x - x.mean()) / x.std())\n",
+ "widths = list(assignment_sample[PS2.Score].apply(lambda x: 0.2 if x < 1 else 3))\n",
+ "plt.scatter(x=subject_times_norm, y=subject_indices, color=colors, linewidths=widths, edgecolors=None)\n",
+ "plt.xlim([-2, 2])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "is_sorted = lambda a: np.all(a[:-1] <= a[1:])\n",
+ "\n",
+ "def is_consistent(rows):\n",
+ " orders = rows[PS2.Order]\n",
+ " times = rows['TimeInt']\n",
+ " return is_sorted(times.values)\n",
+ "\n",
+ "consistent = runs.groupby([PS2.SubjectID, PS2.AssignmentID, PS2.ProblemID]).apply(is_consistent)\n",
+ "consistent"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "runs[(runs[PS2.ProblemID]==102) & (runs[PS2.SubjectID]=='71ffa17407d66e134442eebb32d330ec')]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "consistent[~consistent]"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/verify_split.ipynb b/verify_split.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..4a9dcfb177181f2eda3cf7a3e1e8e173e830f053
--- /dev/null
+++ b/verify_split.ipynb
@@ -0,0 +1,150 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "from ProgSnap2 import ProgSnap2Dataset\n",
+ "from ProgSnap2 import PS2\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "import os\n",
+ "from os import path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "early_assignments = np.array([439, 487, 492])\n",
+ "late_assignments = np.array([494, 502])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def verify_users(path, is_train, users):\n",
+ " subdir = 'Train' if is_train else 'Test'\n",
+ " path = os.path.join(path, subdir)\n",
+ " ps2 = ProgSnap2Dataset(os.path.join(path, 'Data'))\n",
+ " early = pd.read_csv(os.path.join(path, 'early.csv'))\n",
+ " late = pd.read_csv(os.path.join(path, 'late.csv'))\n",
+ " main_table = ps2.get_main_table()\n",
+ " student_table = ps2.load_link_table('Subject')\n",
+ " code_states = ps2.get_code_states_table()\n",
+ " \n",
+ " # Verify the early table\n",
+ " assert early[PS2.SubjectID].isin(users).all()\n",
+ " assert early[PS2.AssignmentID].isin(early_assignments).all()\n",
+ " assert 'Label' in early.columns\n",
+ " assert early['Label'].mean() != 0\n",
+ " \n",
+ " # Verify the late table\n",
+ " assert late[PS2.SubjectID].isin(users).all()\n",
+ " assert late[PS2.AssignmentID].isin(late_assignments).all()\n",
+ " assert ('Label' in late.columns) == is_train\n",
+ " if (is_train):\n",
+ " assert early['Label'].mean() != 0\n",
+ " \n",
+ " # Verify the main table\n",
+ " assert main_table[PS2.SubjectID].isin(users).all()\n",
+ " main_table_assignments = early_assignments\n",
+ " if (is_train):\n",
+ " main_table_assignments = np.append(main_table_assignments, late_assignments)\n",
+ " assert main_table[PS2.AssignmentID].isin(main_table_assignments).all()\n",
+ " \n",
+ " # Verify student table\n",
+ " assert student_table[PS2.SubjectID].isin(users).all()\n",
+ " assert ('X-Grade' in student_table.columns) == is_train\n",
+ " \n",
+ " # Verify the code states table\n",
+ " assert set(code_states[PS2.CodeStateID]) == set(main_table[PS2.CodeStateID])\n",
+ " \n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "FileNotFoundError",
+ "evalue": "[Errno 2] No such file or directory: 'data/Release/S19\\\\split.csv'",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
+ "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_10184/2007109656.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0msemester\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'S19'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'F19'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m \u001b[0msplit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mPATH\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msemester\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'split.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 7\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mf'Train % for {semester} is: {split[\"IsTrain\"].mean()}'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 8\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mf'Count for {semester} is: {len(split.index)}'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\util\\_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m 309\u001b[0m \u001b[0mstacklevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 310\u001b[0m )\n\u001b[1;32m--> 311\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 312\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 313\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[0;32m 584\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 585\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 586\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 587\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 588\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m 480\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 481\u001b[0m \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 482\u001b[1;33m \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 483\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 484\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m 809\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 810\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 811\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 812\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 813\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m 1038\u001b[0m )\n\u001b[0;32m 1039\u001b[0m \u001b[1;31m# error: Too many arguments for \"ParserBase\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1040\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mmapping\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# type: ignore[call-arg]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1041\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1042\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\c_parser_wrapper.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m 49\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 50\u001b[0m \u001b[1;31m# open handles\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_open_handles\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 52\u001b[0m \u001b[1;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandles\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 53\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers\\base_parser.py\u001b[0m in \u001b[0;36m_open_handles\u001b[1;34m(self, src, kwds)\u001b[0m\n\u001b[0;32m 220\u001b[0m \u001b[0mLet\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreaders\u001b[0m \u001b[0mopen\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0mafter\u001b[0m \u001b[0mthey\u001b[0m \u001b[0mare\u001b[0m \u001b[0mdone\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mtheir\u001b[0m \u001b[0mpotential\u001b[0m \u001b[0mraises\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 221\u001b[0m \"\"\"\n\u001b[1;32m--> 222\u001b[1;33m self.handles = get_handle(\n\u001b[0m\u001b[0;32m 223\u001b[0m \u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 224\u001b[0m \u001b[1;34m\"r\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\common.py\u001b[0m in \u001b[0;36mget_handle\u001b[1;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[0;32m 700\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;34m\"b\"\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 701\u001b[0m \u001b[1;31m# Encoding\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 702\u001b[1;33m handle = open(\n\u001b[0m\u001b[0;32m 703\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 704\u001b[0m \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/Release/S19\\\\split.csv'"
+ ]
+ }
+ ],
+ "source": [
+ "PATH = \"data/Release/\"\n",
+ "\n",
+ "last_semester_users = None\n",
+ "\n",
+ "for semester in ['S19', 'F19']:\n",
+ " split = pd.read_csv(os.path.join(PATH, semester, 'split.csv'))\n",
+ " print(f'Train % for {semester} is: {split[\"IsTrain\"].mean()}')\n",
+ " print(f'Count for {semester} is: {len(split.index)}')\n",
+ " for is_train in [True, False]:\n",
+ " path = os.path.join(PATH, semester)\n",
+ " print('Verifying: ' + path)\n",
+ " users = split[split['IsTrain'] == is_train][PS2.SubjectID]\n",
+ " verify_users(path, is_train, users)\n",
+ " \n",
+ " semester_users = split[PS2.SubjectID]\n",
+ " if last_semester_users is not None:\n",
+ " assert len(set(last_semester_users).intersection(semester_users)) == 0\n",
+ " last_semester_users = semester_users"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}