Data Analysis
NumPy arrays
import numpy as np
np.array([1, 2, 3]) # from a list
np.zeros(5) # 0. 0. 0. 0. 0.
np.ones((2, 3)) # 2 rows, 3 columns
np.arange(0, 10, 2) # 0 2 4 6 8 — step known, end excluded
np.linspace(0, 1, 5) # 0. 0.25 0.5 0.75 1. — count known, end included
np.full(3, 7) # 7 7 7
One dtype for the whole array. That is the source of the speed — and of the surprise when an integer array silently becomes a float one.
a.shape # (2, 3)
a.dtype # dtype('int64')
a.ndim # 2
a.size # 6
a.astype(float)
Vectorised arithmetic
a * 2 # every element doubled (a list would repeat itself)
a + b # elementwise, shapes must match or broadcast
a ** 2
np.sqrt(a)
a > 5 # a boolean array, not a single True/False
Selecting
a[0] # one element
a[1:4] # a VIEW — writing to it changes a
a[1:4].copy() # an independent copy
a[a > 5] # the values where the mask is True
a[(a > 2) & (a < 8)] # & and |, each side parenthesised
np.where(a > 5, a, 0) # elementwise if/else, returns a new array
np.where(a > 5) # the indices instead
Aggregation and axis
axis names the dimension being collapsed, not the one kept.
| Call | Result for a 2-D array |
|---|---|
a.sum() | one number for everything |
a.sum(axis=0) | one number per column (rows collapsed) |
a.sum(axis=1) | one number per row (columns collapsed) |
a.mean() a.std() a.min() a.max()
a.argmin() a.argmax() # position, not value
a.cumsum()
Series and DataFrames
import pandas as pd
s = pd.Series([1, 2, 3], index=["a", "b", "c"])
df = pd.DataFrame({"city": ["Oslo", "Lisbon"], "temp": [3, 24]})
A Series is an array plus an index, and arithmetic aligns on that index rather than on position.
First look at anything new
df.shape # (rows, columns)
df.head(3)
df.dtypes # object where you expected a number = something did not parse
df.info()
df.describe() # numeric columns only unless include="all"
df["city"].value_counts() # finds "UK" / "uk" / "U.K."
df.isna().sum() # missing values per column
Columns
df["temp"] # a Series
df[["city", "temp"]] # a DataFrame
df["temp_f"] = df["temp"] * 9 / 5 + 32
df = df.rename(columns={"temp": "temp_c"})
df = df.drop(columns=["humidity"])
loc and iloc
| Looks up by | df.loc[0] after sorting | |
|---|---|---|
.loc | label | whatever row is labelled 0 |
.iloc | position | — |
.iloc[0] | position | always the first row |
They agree on a default index and diverge the moment you filter or sort.
df.loc[3, "city"]
df.loc[df["temp"] > 20, "city"]
df.iloc[0]
df.iloc[0:3, 0:2]
Filtering
df[df["temp"] > 20]
df[(df["temp"] > 20) & (df["city"] != "Oslo")] # & | and parentheses
df[df["city"].isin(["Oslo", "Tokyo"])]
df[df["city"].str.startswith("L")]
df.sort_values("temp", ascending=False)
df.nlargest(3, "temp")
Never df[df.a > 1]["b"] = 0 — that is chained indexing and may assign into a
temporary copy. Write df.loc[df["a"] > 1, "b"] = 0.
Missing data
df.isna().sum()
df.dropna() # rows with any gap
df.dropna(subset=["temp"]) # only where it matters
df["temp"].fillna(df["temp"].mean())
df["team"] = df["team"].fillna("unknown")
Aggregations skip missing values, and mean divides by the count of present
values — so a mostly-empty column reports a confident average of very little.
Grouping
df.groupby("team")["score"].mean()
df.groupby("team")["score"].agg(["mean", "count", "max"])
df.groupby(["team", "role"])["score"].sum()
df.groupby("team").agg(avg=("score", "mean"), n=("score", "size"))
df.groupby("team")["score"].mean().reset_index() # index back to a column
The grouping key becomes the index — hence reset_index().
Joining
pd.merge(orders, customers, on="customer_id") # inner by default
pd.merge(orders, customers, on="customer_id", how="left") # keeps every order
pd.concat([jan, feb], ignore_index=True) # stack rows
Check df.shape before and after every join. Fewer rows means an inner join dropped
unmatched ones; far more means duplicate keys multiplied them.
In and out
pd.read_csv("sales.csv")
pd.read_csv("sales.csv", parse_dates=["date"], na_values=["N/A", "-"])
pd.read_json("data.json")
df.to_csv("out.csv", index=False) # index=False, or you get a mystery column
df.to_dict(orient="records")
Getting plain Python back out
NumPy scalars leak out of pandas very easily and are not quite the types they look like.
float(df["temp"].mean())
int(df["temp"].count())
df["city"].tolist()
df.to_dict(orient="records")