{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'numpy'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m#Numerical Python\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m# To plot\u001b[39;00m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'numpy'"
     ]
    }
   ],
   "source": [
    "#Numerical Python\n",
    "import numpy as np\n",
    "\n",
    "# To plot\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Gamma function\n",
    "from scipy.special import gamma\n",
    "\n",
    "# To calculate statistics\n",
    "from statistics import mode\n",
    "from scipy.stats import norm\n",
    "from scipy.stats import hmean, trim_mean, iqr, median_abs_deviation, skew, kurtosis\n",
    "from scipy.stats.mstats import gmean, winsorize\n",
    "from statsmodels.distributions import ECDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALPHA = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = plt.rcParams['axes.prop_cycle'].by_key()['color']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "Dat = np.array([0.0, 0.8, 1.0, 1.2, 1.3, 1.3, 1.4, 1.8, 2.4, 4.6])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Empirical cumulative distribution function (ECDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ecdf = ECDF(Dat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.05, 4.6499999999999995)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 24608 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Divide the figure in two subfigures\n",
    "f, ax = plt.subplots(2, 1, figsize=(10,6), sharex=True, height_ratios=[1, 3]) # The subfigures share the same x axis\n",
    "\n",
    "# In the first subfigure we show our observations\n",
    "ax[0].set_yticks([])                                        # Do not show the ticks in the vertical axis\n",
    "ax[0].set_ylabel(\"\")                                        # Do not show any label in the vertical axis\n",
    "ax[0].spines[['left', 'right', 'top']].set_visible(False)   # Do not plot the left, right and top margins\n",
    "ax[0].text(2.2, 0.75, \"Data\", fontsize=12)\n",
    "\n",
    "for i in range(len(Dat)):\n",
    "    ax[0].axvline(x=Dat[i], ymax=0.2, c='r')                # Plot the observations as a rugplot\n",
    "\n",
    "# In the second subfigure we show the ECDF\n",
    "ax[1].spines[['right', 'top']].set_visible(False)           # Do not plot the right and top margins\n",
    "\n",
    "# Plot the ECDF\n",
    "ax[1].scatter(ecdf.x, ecdf.y, color='rebeccapurple')        \n",
    "ax[1].step(ecdf.x, ecdf.y, where='post', color='rebeccapurple')\n",
    "\n",
    "# Add title and labels to the plot\n",
    "ax[1].set_title(\"Empirical Cumulative Distribution Function\", fontsize=12)\n",
    "ax[1].set_xlabel(r'$x$')\n",
    "ax[1].set_ylabel(r'$F_n(x)$')\n",
    "\n",
    "# Modify the limits of the vertical and horizontal axes\n",
    "ax[1].set_ylim(0, 1.05)\n",
    "ax[1].set_xlim(Dat.min()-0.05, Dat.max()+0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quantiles, quartiles, percentiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.792    , 1.3689085, 2.422    ])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Quantiles of specific probabilities\n",
    "p = [0.11, 1-np.exp(-1), 0.89]\n",
    "np.quantile(Dat, p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.05, 1.3 , 1.7 ])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Quartiles\n",
    "p = [0.25, 0.5, 0.75]\n",
    "np.quantile(Dat, p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Central tendency statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "Mode = mode(Dat)\n",
    "Median = np.median(Dat)\n",
    "Mean = Dat.mean()\n",
    "GeomMean = gmean(Dat)\n",
    "HarmMean = hmean(Dat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "Weights = norm.pdf(Dat, Dat.mean(), Dat.std()) # Weights for the weighted mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 16664 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the weight for each observation\n",
    "ax = plt.subplot(111)\n",
    "\n",
    "ax.stem(Dat, Weights)\n",
    "ax.set_xlabel('x')\n",
    "ax.set_ylabel('Weight')\n",
    "\n",
    "ax.spines[['right', 'top']].set_visible(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "WeightedMean = np.average(Dat, weights=Weights)\n",
    "TrimmedMean = trim_mean(Dat, 0.1)\n",
    "InterquartileMean = trim_mean(Dat, 0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "masked_array(data=[0.8, 0.8, 1. , 1.2, 1.3, 1.3, 1.4, 1.8, 2.4, 2.4],\n",
       "             mask=False,\n",
       "       fill_value=1e+20)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DatWinsorized = winsorize(Dat, [0.1,0.1]) # Winsorized the data\n",
    "DatWinsorized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "WinsorizedMean = DatWinsorized.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "CentralTendency = [Mode, Median, Mean, GeomMean, HarmMean, WeightedMean, TrimmedMean, InterquartileMean, WinsorizedMean]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(1.75, 0.75, 'Central tendency statistics')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 12104 characters]",
      "text/plain": [
       "<Figure size 1000x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Divide the figure in two subfigures\n",
    "f, ax = plt.subplots(2, 1, figsize=(10,4), sharex=True)     # The subfigures share the same x axis\n",
    "\n",
    "# In the first subfigure we show our observations\n",
    "ax[0].set_yticks([])                                        # Do not show the ticks in the vertical axis\n",
    "ax[0].set_ylabel(\"\")                                        # Do not show any label in the vertical axis\n",
    "ax[0].spines[['left', 'right', 'top']].set_visible(False)   # Do not plot the left, right and top margins\n",
    "ax[0].text(2.2, 0.75, \"Data\", fontsize=12)\n",
    "\n",
    "for i in range(len(Dat)):\n",
    "    ax[0].axvline(x=Dat[i], ymax=0.2, c='r')                # Plot the observations as a rugplot\n",
    "\n",
    "# In the second subfigure we show the central tendency statistics\n",
    "ax[1].spines[['left', 'right', 'top']].set_visible(False)   # Do not plot the left, right and top margins\n",
    "\n",
    "for i in range(len(CentralTendency)):\n",
    "    ax[1].axvline(x=CentralTendency[i], ymax=0.2)\n",
    "\n",
    "ax[1].set_yticks([])\n",
    "ax[1].set_ylabel(\"\")\n",
    "ax[1].text(1.75, 0.75, \"Central tendency statistics\", fontsize=12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DispersionStatistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "S2Biased = np.var(Dat)\n",
    "S2Unbiased = np.var(Dat, ddof=1)\n",
    "\n",
    "S1 = np.sqrt(S2Biased)\n",
    "S2 = np.sqrt(S2Unbiased)\n",
    "R = Dat.max() - Dat.min()\n",
    "IQR = iqr(Dat)\n",
    "MAD = median_abs_deviation(Dat)\n",
    "AAD = abs(Dat-Dat.mean()).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def an(n):\n",
    "    return np.sqrt((n-1)/2) * gamma((n-1)/2) / gamma(n/2)\n",
    "\n",
    "def c4(n):\n",
    "    return 1/an(n)\n",
    "\n",
    "# d2 for n in 2,...,25\n",
    "d2 = {2:1.128, 3:1.693, 4:2.059, 5:2.326, 6:2.534, 7:2.704, 8:2.847, 9:2.970, 10:3.078, 11:3.173, 12:3.258, 13:3.336, 14:3.407, 15:3.472, 16:3.532, 17:3.588, 18:3.640, 19:3.689, 20:3.735, 21:3.778, 22:3.819, 23:3.858, 24:3.895, 25:3.931}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate unbiased estimators for the standard deviation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = len(Dat)\n",
    "\n",
    "sigma_1 = S2 / c4(N)\n",
    "sigma_2 = R / d2[N]\n",
    "sigma_3 = IQR / (2 * norm.ppf(0.75))\n",
    "sigma_4 = MAD / norm.ppf(0.75)\n",
    "sigma_5 = AAD * np.sqrt(np.pi/2)\n",
    "\n",
    "sigma_list = [sigma_1, sigma_2, sigma_3, sigma_4, sigma_5]\n",
    "labels = ['Deviation', 'Range', 'IQR', 'MAD', 'AAD']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we assume that the data follows a normal distribution, a prediction interval of prob. $(1-\\alpha)\\times100\\%$ would have limits corresponding with the quantiles of the normal distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.05, 0.5)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 20984 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "f, ax = plt.subplots(2, 1, figsize=(10,6), sharex=True, height_ratios=[1, 3])\n",
    "\n",
    "for i in range(len(Dat)):\n",
    "    ax[0].axvline(x=Dat[i], ymax=0.2, c='r')\n",
    "\n",
    "ax[0].set_yticks([])\n",
    "ax[0].set_ylabel(\"\")\n",
    "ax[0].spines[['left', 'right', 'top']].set_visible(False)\n",
    "ax[0].text(2, 0.75, \"Data\", fontsize=12)\n",
    "\n",
    "for i, sigma in enumerate(sigma_list):\n",
    "    ax[1].hlines(\n",
    "        0.1*(4-i), \n",
    "        norm.ppf(ALPHA/2, Median, np.sqrt(1+np.pi/(2*N))*sigma), \n",
    "        norm.ppf(1-ALPHA/2, Median, np.sqrt(1+np.pi/(2*N))*sigma),\n",
    "        color=colors[i],\n",
    "        label=labels[i])\n",
    "    ax[1].scatter(Median, 0.1*(4-i))\n",
    "\n",
    "ax[1].legend()\n",
    "ax[1].set_yticks([])\n",
    "ax[1].set_ylabel(\"\")\n",
    "ax[1].spines[['left', 'right', 'top']].set_visible(False)\n",
    "ax[1].text(1.5, 0.45, r\"Prediction intervals, $\\alpha={}$\".format(ALPHA), fontsize=12)\n",
    "ax[1].set_ylim(-0.05, 0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Skewness"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Empirical central moments and k-statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "m2 = S2Biased\n",
    "m3 = ((Dat-Mean)**3).mean()\n",
    "\n",
    "k2 = S2Unbiased\n",
    "k3 = N**2 / ((N-1)*(N-2))*m3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coefficient of skewness based on central moments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "g1_byhand = m3 / m2**(3/2)\n",
    "g1 = skew(Dat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(np.float64(1.472), np.float64(1.472))"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g1_byhand.round(4), g1.round(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coefficient of skewness based on $k$-statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "G1_byhand = k3 / k2**(3/2)\n",
    "G1 = skew(Dat, bias=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(np.float64(1.7455), np.float64(1.7455))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G1_byhand.round(4), G1.round(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Density'>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 29836 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.kdeplot(Dat)\n",
    "\n",
    "for i in range(len(Dat)):\n",
    "    plt.axvline(x=Dat[i], ymax=0.05, c='r')                # Plot the observations as a rugplot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deprecated measures of asymmetry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "Sk1 = (Mean - Mode) / S2\n",
    "Sk2 = (Mean - Median) / S2\n",
    "S75 = (np.quantile(Dat, 0.75) + np.quantile(Dat, 0.25) - 2*np.median(Dat)) / IQR\n",
    "S90 = (np.quantile(Dat, 0.9) + np.quantile(Dat, 0.1) - 2*np.median(Dat)) / (np.quantile(Dat, 0.9) - np.quantile(Dat, 0.1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(np.float64(0.2276),\n",
       " np.float64(0.2276),\n",
       " np.float64(0.2308),\n",
       " np.float64(0.3895))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Sk1.round(4), Sk2.round(4), S75.round(4), S90.round(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kurtosis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "m4 = ((Dat-Mean)**4).mean()\n",
    "\n",
    "k4 = N**2*((N+1)*m4 - 3*(N-1)*m2**2) / ((N-1)*(N-2)*(N-3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coefficient of kurtosis based on central moments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "g2_byhand = m4/m2**2\n",
    "g2 = kurtosis(Dat, fisher=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(np.float64(4.8753), np.float64(4.8753))"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g2_byhand.round(4), g2.round(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coefficient of kurtosis based on $k$-statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "G2_byhand = k4/k2**2 + 3\n",
    "G2 = kurtosis(Dat, fisher=False, bias=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(np.float64(7.2795), np.float64(7.2795))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G2_byhand.round(4), G2.round(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Excess of kurtosis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "g2_excess_byhand = g2_byhand - 3\n",
    "g2_excess = kurtosis(Dat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(np.float64(1.8753), np.float64(1.8753))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g2_excess_byhand.round(4), g2_excess.round(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "G2_excess_byhand = G2_byhand - 3\n",
    "G2_excess = kurtosis(Dat, bias=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(np.float64(4.2795), np.float64(4.2795))"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G2_excess_byhand.round(4), G2_excess.round(4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
