cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)
logreg = LogisticRegression(max_iter=1000, solver='liblinear', random_state=RANDOM_STATE)
def make_pipe(encoder):
ct = ColumnTransformer([
('cat', encoder, encoder_cols),
('num', StandardScaler(), numeric_cols),
])
return Pipeline([('prep', ct), ('model', logreg)])
pipes = {
'OneHotEncoder': make_pipe(OneHotEncoder(handle_unknown='ignore', min_frequency=20)),
'OrdinalEncoder': make_pipe(OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)),
'TargetEncoder cross-fit': make_pipe(TargetEncoder(random_state=RANDOM_STATE, cv=5, smooth=20)),
}
rows = []
for name, pipe in pipes.items():
scores = cross_val_score(pipe, X, y, cv=cv, scoring='roc_auc', n_jobs=None)
rows.append({'encoder': name, 'ROC_AUC_mean': scores.mean(), 'ROC_AUC_std': scores.std()})
print(pd.DataFrame(rows).round(4).to_string(index=False))
# Leakage demo: a derived high-cardinality category from real columns.
leak_cols = ['education', 'detailed_occupation_recode', 'state_of_previous_residence', 'country_of_birth_self']
leak_work = df[leak_cols + numeric_cols + ['income_gt_50k']].sample(n=80000, random_state=RANDOM_STATE + 1).copy()
leak_work[leak_cols] = leak_work[leak_cols].fillna('<missing>').astype(str)
combo = leak_work[leak_cols[0]].astype(str)
for col in leak_cols[1:]:
combo = combo + '|' + leak_work[col].astype(str)
leak_work['category_combo'] = combo
X_combo = leak_work[['category_combo'] + numeric_cols]
y_combo = leak_work['income_gt_50k']
print(f"\nDerived category_combo cardinality: {leak_work['category_combo'].nunique():,}")
# Wrong on purpose: target means computed once on the full sample before CV.
global_mean = y_combo.mean()
full_means = leak_work.groupby('category_combo')['income_gt_50k'].mean()
X_naive = pd.DataFrame({
'category_combo_te': leak_work['category_combo'].map(full_means).fillna(global_mean),
})
for col in numeric_cols:
X_naive[col] = leak_work[col].to_numpy()
naive_pipe = Pipeline([('scaler', StandardScaler()), ('model', logreg)])
naive_scores = cross_val_score(naive_pipe, X_naive, y_combo, cv=cv, scoring='roc_auc')
safe_pipe = Pipeline([
('prep', ColumnTransformer([
('te', TargetEncoder(random_state=RANDOM_STATE, cv=5, smooth=20), ['category_combo']),
('num', StandardScaler(), numeric_cols),
])),
('model', logreg),
])
safe_scores = cross_val_score(safe_pipe, X_combo, y_combo, cv=cv, scoring='roc_auc')
print('\nTarget encoding leakage demo (ROC-AUC):')
print(f" naive pre-CV target mean : {naive_scores.mean():.4f} +/- {naive_scores.std():.4f}")
print(f" cross-fitted pipeline : {safe_scores.mean():.4f} +/- {safe_scores.std():.4f}")
# Unseen-category behavior from year 94 -> 95.
train94 = df[df['year'] == 94].copy()
test95 = df[df['year'] == 95].copy()
train94['country_of_birth_self'] = train94['country_of_birth_self'].fillna('<missing>').astype(str)
test95['country_of_birth_self'] = test95['country_of_birth_self'].fillna('<missing>').astype(str)
ohe = OneHotEncoder(handle_unknown='ignore', sparse_output=False).fit(train94[['country_of_birth_self']])
new_country = sorted(set(test95['country_of_birth_self']) - set(train94['country_of_birth_self']))
print('\nUnseen country_of_birth_self in 1995:', new_country)
if new_country:
examples = test95[test95['country_of_birth_self'].astype(str).isin(new_country)][['country_of_birth_self']].head(3)
encoded = ohe.transform(examples)
print('Jumlah kolom aktif setelah OneHotEncoder(handle_unknown="ignore") untuk contoh unseen:', encoded.sum(axis=1).astype(int).tolist())
