← Back to the demoBack to the start of the demoHome

What is this flat worth? Build an HDB resale price estimator

ITE Higher Nitec in AI Applications: Applied Data Fundamentals, Data Acquisition & Processing, Machine Learning, Responsible AI

In this lab you teach a computer to estimate the resale price of an HDB flat from a few facts about it: town, flat type, floor area, storey and remaining lease. This is regression: the answer is a number (dollars), not a category.

The problem. A family wants a rough idea of what a flat might sell for before they look at listings or talk to an agent. A model trained on past sales can give a first estimate, and say how wrong it usually is.

The data. Housing & Development Board (HDB), Resale flat prices based on registration date from Jan-2017 onwards, data.gov.sg, used under the Singapore Open Data Licence. Every row is a real resale registered with HDB.

Rule for the whole lab. HDB says these prices are indicative only. A model built on them is a teaching tool, not a valuation.

1. Load the data

On the lab machine the file is already downloaded. In Colab, the cell downloads it from data.gov.sg (about 24 MB).

import json
import os
import time
import urllib.request

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

DATASET = "d_8b84c4ee58e3cfc0ece0d773c8ca6abc"
LOCAL_FILE = "../ml/data/raw/hdb/resale-2017-onwards.csv"
COLAB_FILE = "/content/resale-2017-onwards.csv"

def data_file():
    if os.path.exists(LOCAL_FILE):
        return LOCAL_FILE
    if not os.path.exists(COLAB_FILE):
        api = f"https://api-open.data.gov.sg/v1/public/api/datasets/{DATASET}"
        urllib.request.urlopen(f"{api}/initiate-download").read()
        for _ in range(10):
            url = json.load(urllib.request.urlopen(f"{api}/poll-download")).get("data", {}).get("url")
            if url:
                break
            time.sleep(3)
        urllib.request.urlretrieve(url, COLAB_FILE)
    return COLAB_FILE

sales = pd.read_csv(data_file())
print(f"{len(sales):,} resale transactions, {sales.month.min()} to {sales.month.max()}")
sales.head()

244,312 resale transactions, 2017-01 to 2026-11

	month	town	flat_type	block	street_name	storey_range	floor_area_sqm	flat_model	lease_commence_date	remaining_lease	resale_price
0	2017-01	ANG MO KIO	2 ROOM	406	ANG MO KIO AVE 10	10 TO 12	44.0	Improved	1979	61 years 04 months	232000.0
1	2017-01	ANG MO KIO	3 ROOM	108	ANG MO KIO AVE 4	01 TO 03	67.0	New Generation	1978	60 years 07 months	250000.0
2	2017-01	ANG MO KIO	3 ROOM	602	ANG MO KIO AVE 5	01 TO 03	67.0	New Generation	1980	62 years 05 months	262000.0
3	2017-01	ANG MO KIO	3 ROOM	465	ANG MO KIO AVE 10	04 TO 06	68.0	New Generation	1980	62 years 01 month	265000.0
4	2017-01	ANG MO KIO	3 ROOM	601	ANG MO KIO AVE 5	01 TO 03	67.0	New Generation	1980	62 years 05 months	265000.0

2. Clean it with pandas

Three columns are text that a model cannot use as they are:

1. storey_range is a band such as "10 TO 12". We use the middle storey, 11.
2. remaining_lease reads "61 years 04 months". We turn it into years, 61.33.
3. month is "2017-01". We turn it into a date, so we can split by time.

We also check for missing values and duplicates before anything else.

sales["storey"] = sales.storey_range.str.split(" TO ").apply(lambda band: (int(band[0]) + int(band[1])) / 2)
lease = sales.remaining_lease.str.extract(r"(\d+) years?(?: (\d+) months?)?").astype(float).fillna(0)
sales["lease_years"] = lease[0] + lease[1] / 12
sales["date"] = pd.to_datetime(sales.month)

print("Missing values per column:")
print(sales.isna().sum()[sales.isna().sum() > 0].to_string() or "none")
print("Exact duplicate rows:", sales.duplicated().sum(), "(two flats in one block can sell in the same month for the same price, so we keep them)")
sales[["storey_range", "storey", "remaining_lease", "lease_years"]].head(3)

Missing values per column:
Series([], )
Exact duplicate rows: 318 (two flats in one block can sell in the same month for the same price, so we keep them)

	storey_range	storey	remaining_lease	lease_years
0	10 TO 12	11.0	61 years 04 months	61.333333
1	01 TO 03	2.0	60 years 07 months	60.583333
2	01 TO 03	2.0	62 years 05 months	62.416667

3. Explore: what moves the price?

Look before you model. Four questions:

1. How have prices changed over time?
2. How many sales of each flat type are there?
3. Does a bigger flat cost more?
4. Which towns cost most?

by_year = sales.groupby(sales.date.dt.year).resale_price.median()
by_year.plot(marker="o", figsize=(7, 3), title="Median resale price by year ($)")
plt.show()
print(f"Median price {by_year.index[0]}: ${by_year.iloc[0]:,.0f}; {by_year.index[-1]}: ${by_year.iloc[-1]:,.0f} ({by_year.iloc[-1] / by_year.iloc[0] - 1:+.0%})")
print(sales.flat_type.value_counts().to_string())

Median price 2017: $410,000; 2026: $635,000 (+55%)
flat_type
4 ROOM              103792
5 ROOM               59841
3 ROOM               57939
EXECUTIVE            17407
2 ROOM                5151
MULTI-GENERATION        93
1 ROOM                  89

sample = sales.sample(4000, random_state=42)
plt.figure(figsize=(7, 4))
for kind, group in sample.groupby("flat_type"):
    plt.scatter(group.floor_area_sqm, group.resale_price / 1000, s=4, label=kind)
plt.xlabel("floor area (square metres)"); plt.ylabel("price ($ thousands)"); plt.legend(markerscale=4, fontsize=8)
plt.title("Bigger flats cost more, but the spread is wide"); plt.show()
towns = sales[sales.date.dt.year == sales.date.dt.year.max()].groupby("town").resale_price.median().sort_values()
print("Cheapest towns (median, latest year):", ", ".join(f"{t.title()} ${p:,.0f}" for t, p in towns.head(3).items()))
print("Dearest towns (median, latest year):", ", ".join(f"{t.title()} ${p:,.0f}" for t, p in towns.tail(3).items()))

Cheapest towns (median, latest year): Ang Mo Kio $520,000, Jurong East $525,000, Yishun $535,000
Dearest towns (median, latest year): Bishan $850,000, Queenstown $855,000, Bukit Timah $1,000,000

Two cleaning decisions from what you just saw.

1. Prices rise over time, so a model trained on old sales will guess too low today. We will measure exactly how much in section 9.
2. Two flat types (1-room and multi-generation) have so few sales that a model cannot learn them. We leave them out and say so on the model card.

FEW_SALES = 500
counts = sales.flat_type.value_counts()
rare = sorted(counts[counts < FEW_SALES].index)
flats = sales[~sales.flat_type.isin(rare)].copy()
print("Left out (fewer than", FEW_SALES, "sales):", ", ".join(rare), f"-> {len(sales) - len(flats)} rows")
print(f"{len(flats):,} sales kept")

Left out (fewer than 500 sales): 1 ROOM, MULTI-GENERATION -> 182 rows
244,130 sales kept

4. Split by time, not at random

In the scam and recycling labs we split at random. Here that would be cheating: a random split puts sales from next month in the training data, and the model would "know" the future price level.

A real estimator is trained on the past and used on the future, so we test it the same way:

1. Test: the latest 12 months of sales. Locked away until the end.
2. Train: the 12 months before that.

months = sorted(flats.month.unique())
TEST_MONTHS = months[-12:]
TRAIN_MONTHS = months[-24:-12]
train = flats[flats.month.isin(TRAIN_MONTHS)]
test = flats[flats.month.isin(TEST_MONTHS)]
print(f"Train: {len(train):,} sales, {TRAIN_MONTHS[0]} to {TRAIN_MONTHS[-1]}")
print(f"Test:  {len(test):,} sales, {TEST_MONTHS[0]} to {TEST_MONTHS[-1]}")

Train: 25,161 sales, 2024-12 to 2025-11
Test:  24,252 sales, 2025-12 to 2026-11

5. A baseline first

The simplest possible "model": guess the median training price for every flat. Any real model must beat this.

We measure with two numbers, both in the units people care about:

1. MAE (mean absolute error): on average, how many dollars the guess is off by.
2. R squared: the share of the variation in prices the model explains. 1 is perfect; 0 is no better than guessing the average.

from sklearn.metrics import mean_absolute_error, r2_score

baseline_guess = train.resale_price.median()
baseline_mae = mean_absolute_error(test.resale_price, np.full(len(test), baseline_guess))
print(f"Baseline: always guess ${baseline_guess:,.0f}. MAE on the test year: ${baseline_mae:,.0f}")

Baseline: always guess $625,000. MAE on the test year: $167,421

6. Features: turn each flat into numbers

1. Numbers stay numbers: floor area, storey, remaining lease.
2. Categories (town, flat type) become 0/1 columns: one-hot encoding. One town and one flat type are left out as the reference (the first in alphabetical order). Every other town's number then reads "compared with the reference town".

Never number the towns 1, 2, 3: the model would think town 3 is three times town 1.

A trap to avoid. get_dummies(..., drop_first=True) drops whichever town comes first in the rows you give it. On the full training year that is the reference town; on five sample flats it could be any town, and their estimates would silently be wrong. So we fix the reference once, from the training data, and always leave out that same column.

NUMERIC = ["floor_area_sqm", "storey", "lease_years"]
CATEGORIES = ["town", "flat_type"]
REFERENCE = {name: sorted(train[name].unique())[0] for name in CATEGORIES}
REFERENCE_COLUMNS = [f"{name}_{value}" for name, value in REFERENCE.items()]

def features(frame, columns=None):
    encoded = pd.get_dummies(frame[CATEGORIES], dtype=float).join(frame[NUMERIC])
    if columns is None:
        columns = [name for name in encoded.columns if name not in REFERENCE_COLUMNS]
    return encoded.reindex(columns=columns, fill_value=0.0)

X_train = features(train)
COLUMNS = list(X_train.columns)
X_test = features(test, COLUMNS)
y_train, y_test = train.resale_price, test.resale_price
print(X_train.shape[1], "features. Reference town:", REFERENCE["town"].title(), "| reference flat type:", REFERENCE["flat_type"])

32 features. Reference town: Ang Mo Kio | reference flat type: 2 ROOM

7. Linear regression

The model learns one number (a weight) per feature, plus a starting value (the intercept). The estimate is the intercept plus each weight times its feature. That is why we can explain every estimate, line by line.

from sklearn.linear_model import LinearRegression

linear = LinearRegression().fit(X_train, y_train)
weights = pd.Series(linear.coef_, index=COLUMNS)
print(f"Each extra square metre:     ${weights['floor_area_sqm']:+,.0f}")
print(f"Each storey higher:          ${weights['storey']:+,.0f}")
print(f"Each extra year of lease:    ${weights['lease_years']:+,.0f}")
towns_w = weights[weights.index.str.startswith("town_")].sort_values()
print(f"Town, compared with {REFERENCE['town'].title()}: lowest {towns_w.index[0][5:].title()} ${towns_w.iloc[0]:+,.0f}, highest {towns_w.index[-1][5:].title()} ${towns_w.iloc[-1]:+,.0f}")

Each extra square metre:     $+5,440
Each storey higher:          $+5,972
Each extra year of lease:    $+6,371
Town, compared with Ang Mo Kio: lowest Choa Chu Kang $-221,882, highest Bukit Timah $+231,276

8. Test: one honest score, in dollars

pred_linear = linear.predict(X_test)
linear_mae = mean_absolute_error(y_test, pred_linear)
linear_r2 = r2_score(y_test, pred_linear)
print(f"Linear regression on the test year: MAE ${linear_mae:,.0f}, R squared {linear_r2:.3f} (baseline MAE ${baseline_mae:,.0f})")
print(f"Typical size of a miss: {linear_mae / y_test.median():.0%} of the median price")

plt.figure(figsize=(5, 5))
plt.scatter(y_test / 1000, pred_linear / 1000, s=2, alpha=0.3)
line = [y_test.min() / 1000, y_test.max() / 1000]
plt.plot(line, line, color="red"); plt.xlabel("real price ($ thousands)"); plt.ylabel("estimate ($ thousands)")
plt.title("Each dot is one sale. On the red line, the estimate is exact"); plt.show()

Linear regression on the test year: MAE $60,669, R squared 0.850 (baseline MAE $167,421)
Typical size of a miss: 10% of the median price

A stronger model, for comparison

Gradient boosting builds hundreds of small decision trees, each one fixing the last one's mistakes. It can learn curves and combinations (a big flat in one town is not the same as a big flat in another) that a straight line cannot.

from sklearn.ensemble import HistGradientBoostingRegressor

boosted = HistGradientBoostingRegressor(random_state=42).fit(X_train, y_train)
pred_boosted = boosted.predict(X_test)
boosted_mae = mean_absolute_error(y_test, pred_boosted)
boosted_r2 = r2_score(y_test, pred_boosted)
print(f"Gradient boosting: MAE ${boosted_mae:,.0f}, R squared {boosted_r2:.3f}")
print(f"Linear regression: MAE ${linear_mae:,.0f}, R squared {linear_r2:.3f}")

Gradient boosting: MAE $42,754, R squared 0.917
Linear regression: MAE $60,669, R squared 0.850

Which one goes in the web demo? The linear model, on purpose. Gradient boosting is more accurate, but the linear model fits in a small file, runs in a phone's browser with no library, and every estimate can be explained line by line ("+$X for the floor area, -$Y for the shorter lease"). The trade-off between accuracy and explainability is a design decision, and the model card says which we chose and why.

9. Found wrong: data drift

Train the same model on 2017 to 2019 only, then test it on each later year. Nothing about the code changes. Only the world does.

old = flats[flats.date.dt.year <= 2019]
old_model = LinearRegression().fit(features(old, COLUMNS), old.resale_price)
drift = []
for year in range(2020, flats.date.dt.year.max() + 1):
    part = flats[flats.date.dt.year == year]
    guess = old_model.predict(features(part, COLUMNS))
    drift.append({"year": year, "mae": float(mean_absolute_error(part.resale_price, guess)),
                  "underBy": float((part.resale_price - guess).mean()), "sales": int(len(part))})
drift = pd.DataFrame(drift)
drift.plot(x="year", y="mae", marker="o", legend=False, figsize=(7, 3), title="2017-2019 model: average miss by year ($)")
plt.show()
latest = drift.iloc[-1]
print(f"On {int(latest.year)} sales the old model misses by ${latest.mae:,.0f} on average, and guesses ${latest.underBy:,.0f} too low on average.")
print(f"The model trained on the 12 months before the test year misses by ${linear_mae:,.0f}.")

On 2026 sales the old model misses by $245,371 on average, and guesses $245,324 too low on average.
The model trained on the 12 months before the test year misses by $60,669.

What to do about drift. Retrain on recent sales, re-measure on the newest ones, and keep watching the error month by month. The web demo states the months its model was trained on, so a visitor knows how old its view of the market is.

10. Look at the biggest mistakes

errors = test.assign(estimate=pred_linear, miss=pred_linear - y_test)
worst = errors.reindex(errors.miss.abs().sort_values(ascending=False).index).head(5)
print(worst[["month", "town", "flat_type", "floor_area_sqm", "storey_range", "remaining_lease", "resale_price", "estimate"]].round(0).to_string(index=False))
print()
print("Average miss by flat type:")
print(errors.groupby("flat_type").miss.apply(lambda m: m.abs().mean()).round(0).to_string())

  month         town flat_type  floor_area_sqm storey_range    remaining_lease  resale_price  estimate
2026-05  BUKIT MERAH    4 ROOM           150.0     01 TO 03 45 years 09 months     1530000.0  913023.0
2026-11        BEDOK    5 ROOM           113.0     16 TO 18 94 years 06 months     1588000.0 1006596.0
2026-06   QUEENSTOWN    5 ROOM           122.0     04 TO 06 88 years 11 months     1650000.0 1075826.0
2026-09 CENTRAL AREA    5 ROOM           107.0     19 TO 21 83 years 04 months     1720000.0 1152426.0
2026-10   QUEENSTOWN    5 ROOM           111.0     40 TO 42           94 years     1818000.0 1263373.0

Average miss by flat type:
flat_type
2 ROOM       40211.0
3 ROOM       59744.0
4 ROOM       55551.0
5 ROOM       71350.0
EXECUTIVE    67824.0

The worst misses are usually unusual flats: very large, very high, a rare flat model, or a "million-dollar flat" in a central town. The model only knows the five facts it was given. It does not know the view, the renovation or the distance to the MRT.

11. Responsible AI

1. Indicative only. HDB states that resale prices are indicative. An estimate from this model is a teaching example, never a valuation, and the page says so.
2. Show the error. The page shows the estimate with the typical miss (the MAE), never a single exact figure.
3. No personal data. The data has no names. We do not use the block or street in the model, and the web demo asks for no address.
4. State its age. The model card says which months it learned from (data drift).
5. Explainable. Every estimate is the sum of a few lines a person can read and question.

12. Assessment: three short tasks

1. Feature (4 marks). Add one feature you think matters (for example, the year the lease started, or the flat model). Retrain, and report the new test MAE in dollars. Did it help? Why?
2. Drift (4 marks). Train on 2022 to 2023 only. Report the MAE on the test year. Compare it with the 2017-2019 model and the recent model, and explain the order.
3. Model card (4 marks). Write a four-line model card: what it is for, what it is not for, its test MAE, and one situation where it will be badly wrong.

Marking guide (teacher version only)

1. Feature (4 marks). 1 mark for correct code (the new column in both train and test, same columns). 1 mark for the MAE from their own run. 2 marks for a reason tied to the data (for example, flat model captures terrace and DBSS flats that the five facts miss).
2. Drift (4 marks). 1 mark each for the two MAE figures. 2 marks for the explanation: the closer the training months are to the test months, the smaller the drift.
3. Model card (4 marks). 1 mark each: purpose (rough first estimate, teaching); not for (valuation, loans, deciding a price); a correct MAE from their run; a specific failure (an unusual flat, or a market that moves after training).

Appendix: export the model for the web demo

This runs only on the lab machine, where the demo's folders exist. It writes the linear model as plain JSON, every measured figure, a few real test-year sales to try, and the parity cases the demo's tests use to prove the browser gives exactly these estimates. In Colab it does nothing.

WEB_DIR = "../web/clf/price"
PARITY_DIR = "../price/parity"
PARITY_CASES = 200
AREA_PERCENTILES = (0.01, 0.99)

if os.path.isdir(WEB_DIR) and os.path.isdir(PARITY_DIR):
    def options(name):
        return sorted(train[name].unique())
    ranges = {kind: {"area": [round(float(group.floor_area_sqm.quantile(AREA_PERCENTILES[0])), 1), round(float(group.floor_area_sqm.quantile(AREA_PERCENTILES[1])), 1)],
                     "lease": [float(np.floor(group.lease_years.min())), float(np.ceil(group.lease_years.max()))],
                     "typicalArea": float(group.floor_area_sqm.median())}
              for kind, group in train.groupby("flat_type")}
    storeys = sorted(train.storey_range.unique())
    model_out = {
        "format": "linear-regression-dollars/1",
        "trainedOn": {"from": TRAIN_MONTHS[0], "to": TRAIN_MONTHS[-1], "sales": int(len(train))},
        "intercept": round(float(linear.intercept_), 6),
        "numeric": {name: round(float(weights[name]), 6) for name in NUMERIC},
        "town": {name: round(float(weights.get(f"town_{name}", 0.0)), 6) for name in options("town")},
        "flatType": {name: round(float(weights.get(f"flat_type_{name}", 0.0)), 6) for name in options("flat_type")},
        "reference": {"town": REFERENCE["town"], "flatType": REFERENCE["flat_type"]},
        "inputs": {"towns": options("town"), "flatTypes": options("flat_type"), "storeyRanges": storeys, "ranges": ranges},
        "storeyMiddle": "the middle of the storey band, so '10 TO 12' is 11",
        "typicalMissDollars": round(float(linear_mae)),
        "sklearnVersion": __import__("sklearn").__version__,
    }
    metrics = {
        "data": {"sales": int(len(sales)), "kept": int(len(flats)), "leftOut": rare, "first": sales.month.min(), "last": sales.month.max(),
                 "train": {"from": TRAIN_MONTHS[0], "to": TRAIN_MONTHS[-1], "sales": int(len(train))},
                 "test": {"from": TEST_MONTHS[0], "to": TEST_MONTHS[-1], "sales": int(len(test))}},
        "medianByYear": {str(year): float(price) for year, price in by_year.items()},
        "baseline": {"guess": float(baseline_guess), "mae": round(float(baseline_mae))},
        "linear": {"mae": round(float(linear_mae)), "r2": round(float(linear_r2), 4), "missShareOfMedian": round(float(linear_mae / y_test.median()), 4)},
        "boosted": {"mae": round(float(boosted_mae)), "r2": round(float(boosted_r2), 4), "model": "HistGradientBoostingRegressor, default settings"},
        "drift": {"trainedOn": "2017 to 2019", "byYear": [{k: (round(v) if isinstance(v, float) else v) for k, v in row.items()} for row in drift.to_dict("records")]},
        "maeByFlatType": {kind: round(float(v)) for kind, v in errors.groupby("flat_type").miss.apply(lambda m: m.abs().mean()).items()},
    }
    picks = test.groupby("flat_type").sample(1, random_state=42)   # one test-year sale of each flat type, at random, right or wrong
    samples = [{"town": r.town, "flatType": r.flat_type, "floorArea": float(r.floor_area_sqm), "storeyRange": r.storey_range,
                "leaseYears": float(r.lease_years), "month": r.month, "price": float(r.resale_price), "estimate": float(p)}
               for (_, r), p in zip(picks.iterrows(), linear.predict(features(picks, COLUMNS)))]
    rows = test.sample(PARITY_CASES, random_state=7)
    cases = [{"town": r.town, "flatType": r.flat_type, "floorArea": float(r.floor_area_sqm), "storeyRange": r.storey_range,
              "leaseYears": float(r.lease_years), "estimate": float(p)}
             for (_, r), p in zip(rows.iterrows(), linear.predict(features(rows, COLUMNS)))]
    metrics["samples"] = samples
    for path, payload in ((os.path.join(WEB_DIR, "model", "model.json"), model_out), (os.path.join(WEB_DIR, "model", "metrics.json"), metrics),
                          (os.path.join(PARITY_DIR, "parity.json"), {"note": "Estimates from the notebook for test-year HDB resale sales (data.gov.sg, Singapore Open Data Licence).", "cases": cases})):
        os.makedirs(os.path.dirname(path), exist_ok=True)
        with open(path, "w", encoding="utf-8") as handle:
            json.dump(payload, handle, separators=(",", ":"))
    print(f"Exported: model.json, metrics.json, {len(samples)} sample sales, {len(cases)} parity cases")
else:
    print("Not on the lab machine: nothing exported.")

Exported: model.json, metrics.json, 5 sample sales, 200 parity cases