Alert when completion drops

Goal: a Slack message when a quiz’s completion rate this week is well below last week’s, and a short morning digest of yesterday’s numbers.

Uses: Analytics. Scope: analytics:read (quizzes:read too if you list the quizzes with GET /v1/quizzes). One timezone, America/New_York below, decides “today” and goes as tz on every request, so both weeks and the digest describe the same days.

The alert

GET /v1/quizzes/{quiz_id}/analytics/overview answers totals.completion_rate for an inclusive date range. Ask for two disjoint weeks and compare:

Python

import datetime as dt
import os
from zoneinfo import ZoneInfo
import httpx

HEADERS = {"Authorization": f"Bearer {os.environ['OCTANE_API_KEY']}"}
QUIZ = "https://api.octaneai.com/v1/quizzes/quiz_7c9e6679742540de944be07fc1f90ae7"
SLACK = os.environ["SLACK_WEBHOOK_URL"]
TZ = "America/New_York"  # one timezone for "today" and for every request

def totals(start, end):
    r = httpx.get(f"{QUIZ}/analytics/overview", params={"from": start, "to": end, "tz": TZ}, headers=HEADERS, timeout=30)
    r.raise_for_status()
    return r.json()["totals"]

today = dt.datetime.now(ZoneInfo(TZ)).date()
this_week = totals(today - dt.timedelta(days=7), today - dt.timedelta(days=1))
last_week = totals(today - dt.timedelta(days=14), today - dt.timedelta(days=8))

now, before = this_week["completion_rate"], last_week["completion_rate"]  # percentages, 62.5
print(this_week["starts"], now, before)
if this_week["starts"] >= 50 and now is not None and before and (before - now) / before > 0.2:
    httpx.post(SLACK, json={"text": (
        f"Quiz completion is down {(before - now) / before:.0%} week over week: "
        f"{now:.1f}% of {this_week['starts']} starts, was {before:.1f}%."
    )}, timeout=30).raise_for_status()

TypeScript

const HEADERS = { Authorization: `Bearer ${process.env.OCTANE_API_KEY}` };
const QUIZ = "https://api.octaneai.com/v1/quizzes/quiz_7c9e6679742540de944be07fc1f90ae7";
const SLACK = process.env.SLACK_WEBHOOK_URL!;
const TZ = "America/New_York"; // one timezone for "today" and for every request

const day = (offset: number) =>
  new Intl.DateTimeFormat("en-CA", { timeZone: TZ }).format(new Date(Date.now() - offset * 86400000)); // YYYY-MM-DD
async function totals(from: string, to: string) {
  const qs = new URLSearchParams({ from, to, tz: TZ });
  const res = await fetch(`${QUIZ}/analytics/overview?${qs}`, { headers: HEADERS });
  if (!res.ok) throw new Error(`${res.status} ${await res.text()}`);
  return (await res.json()).totals;
}

const thisWeek = await totals(day(7), day(1));
const lastWeek = await totals(day(14), day(8));
const now = thisWeek.completion_rate, before = lastWeek.completion_rate; // percentages, 62.5
console.log(thisWeek.starts, now, before);
if (thisWeek.starts >= 50 && now !== null && before && (before - now) / before > 0.2) {
  await fetch(SLACK, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ text: `Quiz completion is down ${Math.round(((before - now) / before) * 100)}% week over week: ${now.toFixed(1)}% of ${thisWeek.starts} starts, was ${before.toFixed(1)}%.` }),
  });
}

Go

package main

import (
  "bytes"
  "encoding/json"
  "fmt"
  "log"
  "net/http"
  "net/url"
  "os"
  "time"
)

const QUIZ = "https://api.octaneai.com/v1/quizzes/quiz_7c9e6679742540de944be07fc1f90ae7"
const TZ = "America/New_York" // one timezone for "today" and for every request

var loc, _ = time.LoadLocation(TZ)

type Totals struct {
  Starts         int      `json:"starts"`
  Completions    int      `json:"completions"`
  CompletionRate *float64 `json:"completion_rate"`
}

func day(offset int) string { return time.Now().In(loc).AddDate(0, 0, -offset).Format("2006-01-02") }

func totals(from, to string) Totals {
  q := url.Values{"from": {from}, "to": {to}, "tz": {TZ}}
  req, _ := http.NewRequest("GET", QUIZ+"/analytics/overview?"+q.Encode(), nil)
  req.Header.Set("Authorization", "Bearer "+os.Getenv("OCTANE_API_KEY"))
  res, err := http.DefaultClient.Do(req)
  if err != nil {
    log.Fatal(err)
  }
  defer res.Body.Close()
  if res.StatusCode != 200 {
    log.Fatalf("status %d", res.StatusCode)
  }
  var body struct {
    Totals Totals `json:"totals"`
  }
  json.NewDecoder(res.Body).Decode(&body)
  return body.Totals
}

func main() {
  thisWeek := totals(day(7), day(1))
  lastWeek := totals(day(14), day(8))
  fmt.Println(thisWeek.Starts, thisWeek.CompletionRate, lastWeek.CompletionRate)
  if thisWeek.Starts >= 50 && thisWeek.CompletionRate != nil && lastWeek.CompletionRate != nil && *lastWeek.CompletionRate > 0 {
    now, before := *thisWeek.CompletionRate, *lastWeek.CompletionRate // percentages, 62.5
    if (before-now)/before > 0.2 {
      msg, _ := json.Marshal(map[string]string{"text": fmt.Sprintf(
        "Quiz completion is down %.0f%% week over week: %.1f%% of %d starts, was %.1f%%.",
        (before-now)/before*100, now, thisWeek.Starts, before)})
      http.Post(os.Getenv("SLACK_WEBHOOK_URL"), "application/json", bytes.NewReader(msg))
    }
  }
}

• Ranges are inclusive on both ends, so keep the two weeks disjoint.
• completion_rate is a percentage with two decimals (62.5); it is null when nothing started.
• Require a minimum number of starts before alerting; a quiz with ten starts swings wildly.

The digest

Yesterday’s numbers are one call with from and to set to the same day, plus the top products, both in the same tz:

Python

import datetime as dt
import os
from zoneinfo import ZoneInfo
import httpx

HEADERS = {"Authorization": f"Bearer {os.environ['OCTANE_API_KEY']}"}
QUIZ = "https://api.octaneai.com/v1/quizzes/quiz_7c9e6679742540de944be07fc1f90ae7"
SLACK = os.environ["SLACK_WEBHOOK_URL"]
TZ = "America/New_York"

yesterday = (dt.datetime.now(ZoneInfo(TZ)).date() - dt.timedelta(days=1)).isoformat()
r = httpx.get(f"{QUIZ}/analytics/overview", params={"from": yesterday, "to": yesterday, "tz": TZ}, headers=HEADERS, timeout=30)
r.raise_for_status()
t = r.json()["totals"]
r = httpx.get(f"{QUIZ}/analytics/products", params={"from": yesterday, "to": yesterday, "tz": TZ, "sort": "revenue", "limit": 5}, headers=HEADERS, timeout=30)
r.raise_for_status()
products = r.json()

revenue = f"{t['revenue']['amount']} {t['revenue']['currency']}" if t["revenue"] else "no revenue"  # money can be null
lines = [f"*{yesterday}*: {t['views']} views, {t['starts']} starts, {t['completions']} completions, {t['orders']} orders, {revenue}"]
for p in products["rows"]:
    lines.append(f"- {p['title']}: shown {p['times_shown']}x, {p['line_revenue']['amount']} {p['line_revenue']['currency']}")
print("\n".join(lines))
httpx.post(SLACK, json={"text": "\n".join(lines)}, timeout=30).raise_for_status()

TypeScript

const HEADERS = { Authorization: `Bearer ${process.env.OCTANE_API_KEY}` };
const QUIZ = "https://api.octaneai.com/v1/quizzes/quiz_7c9e6679742540de944be07fc1f90ae7";
const SLACK = process.env.SLACK_WEBHOOK_URL!;
const TZ = "America/New_York";

const yesterday = new Intl.DateTimeFormat("en-CA", { timeZone: TZ }).format(new Date(Date.now() - 86400000)); // YYYY-MM-DD
const get = async (path: string, params: Record<string, string>) => {
  const res = await fetch(`${QUIZ}${path}?${new URLSearchParams(params)}`, { headers: HEADERS });
  if (!res.ok) throw new Error(`${res.status} ${await res.text()}`);
  return res.json();
};
const t = (await get("/analytics/overview", { from: yesterday, to: yesterday, tz: TZ })).totals;
const products = await get("/analytics/products", { from: yesterday, to: yesterday, tz: TZ, sort: "revenue", limit: "5" });

const revenue = t.revenue ? `${t.revenue.amount} ${t.revenue.currency}` : "no revenue"; // money can be null
const lines = [`*${yesterday}*: ${t.views} views, ${t.starts} starts, ${t.completions} completions, ${t.orders} orders, ${revenue}`];
for (const p of products.rows) lines.push(`- ${p.title}: shown ${p.times_shown}x, ${p.line_revenue.amount} ${p.line_revenue.currency}`);
console.log(lines.join("\n"));
await fetch(SLACK, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text: lines.join("\n") }) });

Go

package main

import (
  "bytes"
  "encoding/json"
  "fmt"
  "log"
  "net/http"
  "net/url"
  "os"
  "strings"
  "time"
)

const QUIZ = "https://api.octaneai.com/v1/quizzes/quiz_7c9e6679742540de944be07fc1f90ae7"
const TZ = "America/New_York"

type Money struct {
  Amount   string `json:"amount"`
  Currency string `json:"currency"`
}

func get(path string, q url.Values, into any) {
  req, _ := http.NewRequest("GET", QUIZ+path+"?"+q.Encode(), nil)
  req.Header.Set("Authorization", "Bearer "+os.Getenv("OCTANE_API_KEY"))
  res, err := http.DefaultClient.Do(req)
  if err != nil {
    log.Fatal(err)
  }
  defer res.Body.Close()
  if res.StatusCode != 200 {
    log.Fatalf("status %d", res.StatusCode)
  }
  json.NewDecoder(res.Body).Decode(into)
}

func main() {
  loc, _ := time.LoadLocation(TZ)
  yesterday := time.Now().In(loc).AddDate(0, 0, -1).Format("2006-01-02")
  var overview struct {
    Totals struct {
      Views, Starts, Completions, Orders int
      Revenue                            *Money `json:"revenue"` // money can be null
    } `json:"totals"`
  }
  get("/analytics/overview", url.Values{"from": {yesterday}, "to": {yesterday}, "tz": {TZ}}, &overview)
  var products struct {
    Rows []struct {
      Title       *string `json:"title"`
      TimesShown  int     `json:"times_shown"`
      LineRevenue Money   `json:"line_revenue"`
    } `json:"rows"`
  }
  get("/analytics/products", url.Values{"from": {yesterday}, "to": {yesterday}, "tz": {TZ}, "sort": {"revenue"}, "limit": {"5"}}, &products)

  t := overview.Totals
  revenue := "no revenue"
  if t.Revenue != nil {
    revenue = t.Revenue.Amount + " " + t.Revenue.Currency
  }
  lines := []string{fmt.Sprintf("*%s*: %d views, %d starts, %d completions, %d orders, %s", yesterday, t.Views, t.Starts, t.Completions, t.Orders, revenue)}
  for _, p := range products.Rows {
    lines = append(lines, fmt.Sprintf("- %v: shown %dx, %s %s", *p.Title, p.TimesShown, p.LineRevenue.Amount, p.LineRevenue.Currency))
  }
  fmt.Println(strings.Join(lines, "\n"))
  msg, _ := json.Marshal(map[string]string{"text": strings.Join(lines, "\n")})
  http.Post(os.Getenv("SLACK_WEBHOOK_URL"), "application/json", bytes.NewReader(msg))
}

Money is {"amount": "12.50", "currency": "USD"} with a decimal string, never a float; format it, do not add it up as a float.

Things to know

• Cost. An analytics body costs 3 units; these few calls a day are nothing against a Plus allowance of 72 with 12 refilled per second. A burst over many quizzes should read remaining in the RateLimit header and sleep on a 429 for Retry-After seconds.
• Cache. Answers are cached up to 300 seconds on our side and carry an ETag; a job that re-reads the same range can send If-None-Match and pay 1 unit for a 304.
• Freshness. freshness.closed_days_through says the last day that will not change any more; yesterday is closed by the time a morning job runs, today is read live.
• Every quiz. To watch all published quizzes, walk GET /v1/quizzes?status=published (needs quizzes:read) and run the alert per id, or read GET /v1/analytics/quizzes?sort=completion_rate&dir=asc for the whole store (25 quizzes a page by default, limit up to 1000).