Wikipediaで測る「翻訳ラグ」— 英語発の技術トピックが日本語圏へ届くまで

https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/{project}/{access}/{agent}/{article}/daily/{start}/{end}
# 例: .../per-article/en.wikipedia/all-access/user/WebGPU/daily/20200101/20251231

import urllib.parse, time, requests, pandas as pd

HEADERS = {"User-Agent": "TranslationLag/1.0 (contact@example.com)"}
BASE = "https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article"
PAIRS = {"WebGPU": ("WebGPU", "WebGPU"),
         "Rust": ("Rust_(programming_language)", "Rust_(プログラミング言語)")}

def fetch(project, article, start="20200101", end="20251231", retries=3):
    art = urllib.parse.quote(article, safe="")
    # "pagination": API caps long ranges, so request per-year and concatenate.
    frames = []
    for yr in range(int(start[:4]), int(end[:4]) + 1):
        lo, hi = max(start, f"{yr}0101"), min(end, f"{yr}1231")
        url = f"{BASE}/{project}/all-access/user/{art}/daily/{lo}/{hi}"
        for attempt in range(retries):
            try:
                r = requests.get(url, headers=HEADERS, timeout=30)
                if r.status_code == 404:
                    break  # no data for this slice
                r.raise_for_status()
                frames.append(pd.DataFrame(r.json()["items"]))
                break
            except requests.RequestException:
                if attempt == retries - 1:
                    raise
                time.sleep(2 ** attempt)
    if not frames:
        return pd.DataFrame(columns=["date", "views"])
    df = pd.concat(frames, ignore_index=True)
    df["date"] = pd.to_datetime(df["timestamp"], format="%Y%m%d%H")
    return df[["date", "views"]]

if __name__ == "__main__":
    rows = []
    for topic, (en, ja) in PAIRS.items():
        en_df = fetch("en.wikipedia", en)
        ja_df = fetch("ja.wikipedia", ja)
        en_df.to_csv(f"data/{topic}_en.csv", index=False)
        ja_df.to_csv(f"data/{topic}_ja.csv", index=False)
        rows.append({"topic": topic, "en_rows": len(en_df), "ja_rows": len(ja_df)})
    pd.DataFrame(rows).to_csv("data/manifest.csv", index=False)

import numpy as np
import pandas as pd
from scipy import stats

rng = np.random.default_rng(42)
N = 120

# Synthetic EN/JA matched tech-topic article pairs
en_year = rng.integers(2010, 2024, N)
# Salience: English first-year pageviews (log-normal) and edit counts
en_pageviews = rng.lognormal(mean=10, sigma=1.2, size=N)
en_edits = rng.poisson(lam=np.clip(en_pageviews / 5000, 2, 200))

# Lag generation: right-skewed (gamma), shrinks with salience and over time
base = 8.5
salience_eff = -1.3 * (np.log(en_pageviews) - np.log(en_pageviews).mean())
year_eff = -0.6 * (en_year - 2018)
mean_lag = np.clip(base + salience_eff + year_eff, 0.5, None)
lag_months = rng.gamma(shape=2.0, scale=mean_lag / 2.0)  # right-skew

df = pd.DataFrame(dict(en_year=en_year, en_pageviews=en_pageviews,
                       en_edits=en_edits, lag_months=lag_months))

# H1: central tendency + skew, one-sample test vs 8.5 months
mean_, med_, skew_ = df.lag_months.mean(), df.lag_months.median(), stats.skew(df.lag_months)
t1, p1 = stats.ttest_1samp(df.lag_months, 8.5)
print(f"H1 mean={mean_:.2f} median={med_:.2f} skew={skew_:.2f} (t vs 8.5: t={t1:.2f}, p={p1:.3g})")
print("H1 median in [4,8]?", 4 <= med_ <= 8)

# H2: lag ~ log(pageviews) + log(edits), expect negative coefficients
X2 = pd.DataFrame(dict(const=1.0, log_views=np.log(df.en_pageviews), log_edits=np.log(df.en_edits)))
beta2, *_ = np.linalg.lstsq(X2.values, df.lag_months.values, rcond=None)
r2 = stats.pearsonr(np.log(df.en_pageviews), df.lag_months)
print(f"H2 beta(log_views)={beta2[1]:.2f} beta(log_edits)={beta2[2]:.2f} r(views,lag)={r2.statistic:.2f} p={r2.pvalue:.3g}")

# H3: pre/post-2018 group difference
pre, post = df[df.en_year < 2018].lag_months, df[df.en_year >= 2018].lag_months
t3, p3 = stats.ttest_ind(pre, post, equal_var=False)
print(f"H3 pre={pre.mean():.2f} post={post.mean():.2f} (t={t3:.2f}, p={p3:.3g})")

git clone https://github.com/example/wiki-translation-lag.git
cd wiki-translation-lag
python -m venv .venv && source .venv/bin/activate
pip install -r requirements.txt   # requests, pandas, numpy, scipy
python fetch.py        # data/ にEN/JAペアのCSVを取得
python analyze.py      # H1–H3の検定結果を標準出力に表示