Event Contributor Analysis¶

# @replace DATA_SOURCES
DATA_SOURCES = {'badges': '/data/badges', 'datagrepper-raw': '/data/datagrepper-raw', 'datagrepper-parse-accounts': '/data/datagrepper-parse-accounts'}
parquet_dir = DATA_SOURCES["datagrepper-parse-accounts"]
badge_data = DATA_SOURCES["badges"] + "/badge.csv"

cutoff_date = (pd.Timestamp.now().replace(day=1) - pd.DateOffset(weeks=52)).date()

files = []
for p in Path(parquet_dir).glob("fedora-*.parquet"):
    stem = p.stem.replace("_processed", "")
    d = datetime.strptime(stem.split("-")[1], "%Y%m%d").date()
    if d >= cutoff_date:
        files.append(str(p))

dataset = ds.dataset(files, format="parquet")

chunks = []
for batch in dataset.to_batches(batch_size=50_000):
    df = batch.to_pandas()
    if "sent_at" not in df.columns or "username" not in df.columns:
        continue
    df["sent_at"] = pd.to_datetime(df["sent_at"], errors="coerce").dt.floor("s")
    chunks.append(df)

combined_df = pd.concat(chunks, ignore_index=True) if chunks else pd.DataFrame()

if not combined_df.empty:
    print("Maximum date in data:", combined_df["sent_at"].max().date())
    print("Minimum date in data:", combined_df["sent_at"].min().date())
else:
    print("No data found in cutoff range")

activity = combined_df

# Get account creation times
fas_df = activity[activity["topic"] == "org.fedoraproject.prod.fas.user.create"]
account_ages = fas_df.groupby("username")["sent_at"].min().reset_index()
account_ages.columns = ["username", "account_created"]


# Load badge CSV
badge_df = pd.read_csv(badge_data, parse_dates=["timestamp"])
print("Latest Badges data timestamp:", badge_df["timestamp"].max())
badge_df.rename(columns={"fas": "username"}, inplace=True)
badge_df["event"] = badge_df["badge_id"].map(event_badges)
badge_df.dropna(subset=["event", "timestamp"], inplace=True)

print("Latest Event Badges data timestamp:", badge_df["timestamp"].max())

# Add helper column
df["contributor_type"] = df["newcomer_30d"].map({True: "Newcomer", False: "Existing"})

# Extract event year for sorting
df["event_year"] = df["event"].str.extract(r"(\d{4})").astype(int)

# Count newcomers and existing by event
counts = df.groupby(["event", "event_year", "contributor_type"])["username"].count().reset_index()
pivot = counts.pivot(index=["event", "event_year"], columns="contributor_type", values="username").fillna(0)
pivot = pivot.sort_values("event_year")

# Plot
pivot.plot(kind="bar", stacked=True, figsize=(14, 6), color=["lightblue", "steelblue"])
plt.ylabel("Number of Contributors")
plt.title("Event Attendance: Newcomers vs Existing Contributors")
plt.xticks(rotation=45, ha="right")
plt.tight_layout()
plt.show()

activity_cols = [f"M{-i}" for i in range(4, 0, -1)] + [f"M+{i}" for i in range(1, 5)]

# Melt into long format for time series plotting
melted = df.melt(
    id_vars=["event", "event_year", "contributor_type"],
    value_vars=activity_cols,
    var_name="month",
    value_name="msg_count"
)

# Aggregate average msg_count
activity_summary = (
    melted.groupby(["event", "contributor_type", "month"])["msg_count"]
    .mean()
    .reset_index()
)

# Sort months
activity_summary["month"] = pd.Categorical(
    activity_summary["month"],
    categories=activity_cols,
    ordered=True
)

# Plot
g = sns.FacetGrid(
    activity_summary,
    row="event",
    hue="contributor_type",
    height=2.5,
    aspect=3,
    sharey=False,
    palette={"Newcomer": "lightblue", "Existing": "steelblue"}
)
g.map(sns.lineplot, "month", "msg_count")
g.add_legend(title="Contributor Type")
g.set_axis_labels("Month Offset from Event", "Avg. Messages")
g.set_titles("{row_name}")
g.fig.subplots_adjust(top=0.95)
g.fig.suptitle("Activity Profiles: Newcomers vs Existing Contributors", fontsize=16)
plt.show()

return_rate = df.groupby("event")["returned_next_year"].mean().sort_values(ascending=False) * 100

plt.figure(figsize=(12, 6))
sns.barplot(x=return_rate.index, y=return_rate.values)
plt.xticks(rotation=45, ha="right")
plt.ylabel("Return Rate (%)")
plt.title("Event-to-Event Return Engagement")
plt.tight_layout()
plt.show()

activity_columns = [f"M{-i}" for i in range(4, 0, -1)] + [f"M+{i}" for i in range(1, 5)]
monthly_profile = df.groupby("event")[activity_columns].mean()

plt.figure(figsize=(14, 8))
sns.heatmap(monthly_profile, annot=True, fmt=".1f", cmap="rocket_r", linewidths=0.5, linecolor="gray")
plt.title("Avg. Monthly Activity Offset by Event")
plt.xlabel("Month Offset from Event")
plt.ylabel("Event")
plt.tight_layout()
plt.show()