diff --git a/patterns/echodefs.sty b/patterns/echodefs.sty new file mode 100644 index 0000000..ac5ef8c --- /dev/null +++ b/patterns/echodefs.sty @@ -0,0 +1,257 @@ +\ProvidesPackage{echodefs} + + +%----------------------------------------------------------------------------- +% PROCESS PACKAGE OPTIONS +%----------------------------------------------------------------------------- + +\DeclareOption{matit}{ + \providecommand{\vect}[1]{\ensuremath{\bm{{#1}}}} + \providecommand{\mat}[1]{\ensuremath{\bm{{#1}}}} +} + +\DeclareOption{matnotit}{ + \providecommand{\vect}[1]{\ensuremath{\mathbf{#1}}} + \providecommand{\mat}[1]{\ensuremath{\mathbf{#1}}} +} + +\ExecuteOptions{matit} +\ProcessOptions\relax + +%----------------------------------------------------------------------------- +% EXTERNAL PACKAGES +%----------------------------------------------------------------------------- + +% Common commands and definitions + +\RequirePackage{amsmath} +\RequirePackage{amssymb} +\RequirePackage{amsthm} +\RequirePackage{latexsym} +\RequirePackage{verbatim} +\RequirePackage{graphicx} +\RequirePackage{algorithm} +\RequirePackage{algpseudocode} +\RequirePackage{enumerate} +\RequirePackage{bm} +\RequirePackage{hyperref} +\RequirePackage{mathtools} +% \RequirePackage{xcolor} +\RequirePackage{tikz} +\RequirePackage[framemethod=tikz]{mdframed} +\RequirePackage{multicol} + +%----------------------------------------------------------------------------- +% CUSTOM ENVIRONMENTS +%----------------------------------------------------------------------------- + +\newtheorem{theorem}{Theorem} +% \newtheorem{proposition}{Proposition} +\newtheorem{lemma}{Lemma} +\newtheorem{corollary}{Corollary} +\newtheorem{proposition}{Proposition} +\newtheorem{problem}{Problem} +\newtheorem{definition}{Definition} + +% \newenvironment{eqali}{ +% \begin{equation} +% \begin{aligned} +% } +% { +% \end{aligned} +% \end{equation} +% } + +%----------------------------------------------------------------------------- +% REFERENCING AND CITING COMMANDS +%----------------------------------------------------------------------------- + +\providecommand{\eref}[1]{\eqref{eq:#1}} +\providecommand{\cref}[1]{Chapter~\ref{chap:#1}} +\providecommand{\sref}[1]{Section~\ref{sec:#1}} +\providecommand{\fref}[1]{Figure~\ref{fig:#1}} +\providecommand{\sfref}[1]{Fig.~\ref{subfig:#1})} +\providecommand{\tref}[1]{Table~\ref{tab:#1}} +\providecommand{\aref}[1]{Algorithm~\ref{alg:#1}} +\providecommand{\lref}[1]{Lemma~\ref{lem:#1}} +\providecommand{\thref}[1]{Theorem~\ref{thm:#1}} + +%----------------------------------------------------------------------------- +% SET SYMBOLS +%----------------------------------------------------------------------------- + +\providecommand{\RR}{\ensuremath{\mathbb{R}}} +\providecommand{\CC}{\ensuremath{\mathbb{C}}} +\providecommand{\NN}{\ensuremath{\mathbb{N}}} +\renewcommand{\SS}{\ensuremath{\mathbb{S}}} +\providecommand{\ZZ}{\ensuremath{\mathbb{Z}}} +\providecommand{\PP}{\ensuremath{\mathbb{P}}} +\providecommand{\QQ}{\ensuremath{\mathbb{Q}}} +\providecommand{\EE}{\ensuremath{\mathbb{E}}} + +%----------------------------------------------------------------------------- +% MATH MACROS +%----------------------------------------------------------------------------- + +\providecommand{\abs}[1]{{\left|#1\right|}} +\providecommand{\norm}[1]{\left\lVert#1\right\rVert} +\providecommand{\expect}[1]{\mathbb{E}\left[#1\right]} +\providecommand{\bignorm}[1]{\left\lVert#1\right\rVert} +\providecommand{\inprod}[1]{\left\langle#1\right\rangle} +\providecommand{\set}[1]{\left\{#1\right\}} +\providecommand{\seq}[1]{\left<#1\right>} +\providecommand{\bydef}{\overset{\text{def}}{=}} +\providecommand{\diag}{\mathop{\mathrm{diag}}} +\providecommand{\rank}{\mathop{\mathrm{rank}}} +\providecommand{\conv}{\ast} +\providecommand{\circonv}[1]{\circledast_{#1}} +\providecommand{\parder}[2]{{\partial{#1} \over \partial{#2}}} +\providecommand{\parderr}[2]{{\partial^2{#1} \over \partial{#2}^2}} +\providecommand{\Rint}{\int_{\R}} +\providecommand{\di}{\ensuremath{\mathrm{d}}} +\providecommand{\e}{\ensuremath{\mathrm{e}}} +\providecommand{\I}{\ensuremath{\mathrm{j}}} +\providecommand{\epi}[1]{\e^{\hspace{1pt}\I{#1}}} +\providecommand{\eni}[1]{\e^{-\I{#1}}} +\providecommand{\T}{\ensuremath{T}} +\providecommand{\rect}{\ensuremath{\mathrm{rect}}} +\providecommand{\tri}{\ensuremath{\mathrm{tri}}} +\providecommand{\sign}{\ensuremath{\mathrm{sign}}} + +\DeclareMathOperator*{\argmin}{arg\,min} +\DeclareMathOperator*{\argmax}{arg\,max} +\DeclareMathOperator*{\minimize}{minimize} +\DeclareMathOperator*{\maximize}{maximize} + + +% Functions +\providecommand{\tri}{\ensuremath{\mathrm{tri}}} + + +% Vector analysis +% \renewcommand{\div}{\nabla} % Conflicts with mathabx +\providecommand{\grad}{\nabla} + +% Some calligraphic letters +\providecommand{\calS}{\mathcal{S}} +\providecommand{\calP}{\mathcal{P}} +\providecommand{\calR}{\mathcal{R}} +\providecommand{\calH}{\mathcal{H}} +\providecommand{\calM}{\mathcal{M}} +\providecommand{\calV}{\mathcal{V}} +\providecommand{\calF}{\mathcal{F}} +\providecommand{\calG}{\mathcal{G}} +\providecommand{\calO}{\mathcal{O}} +\providecommand{\calX}{\mathcal{X}} +\providecommand{\calY}{\mathcal{Y}} +\providecommand{\calL}{\mathcal{L}} +\providecommand{\calJ}{\mathcal{J}} +\providecommand{\calN}{\mathcal{N}} +\providecommand{\calU}{\mathcal{U}} +\providecommand{\calT}{\mathcal{T}} +\providecommand{\calI}{\mathcal{I}} +\providecommand{\calC}{\mathcal{C}} +\providecommand{\calD}{\mathcal{D}} + + +%----------------------------------------------------------------------------- +% MATRICES AND VECTORS +%----------------------------------------------------------------------------- + +\providecommand{\wh}[1]{\ensuremath{\widehat{#1}}} +\providecommand{\wt}[1]{\ensuremath{\widetilde{#1}}} + +\providecommand{\mA}{\mat{A}} +\providecommand{\mB}{\mat{B}} +\providecommand{\mC}{\mat{C}} +\providecommand{\mD}{\mat{D}} +\providecommand{\mE}{\mat{E}} +\providecommand{\mF}{\mat{F}} +\providecommand{\mH}{\mat{H}} +\providecommand{\mI}{\mat{I}} +\providecommand{\mJ}{\mat{J}} +\providecommand{\mK}{\mat{K}} +\providecommand{\mL}{\mat{L}} +\providecommand{\mN}{\mat{N}} +\providecommand{\mM}{\mat{M}} +\providecommand{\mO}{\mat{O}} +\providecommand{\mP}{\mat{P}} +\providecommand{\mQ}{\mat{Q}} +\providecommand{\mR}{\mat{R}} +\providecommand{\mS}{\mat{S}} +\providecommand{\mU}{\mat{U}} +\providecommand{\mV}{\mat{V}} +\providecommand{\mT}{\mat{T}} +\providecommand{\mW}{\mat{W}} +\providecommand{\mG}{\mat{G}} +\providecommand{\mX}{\mat{X}} +\providecommand{\mY}{\mat{Y}} +\providecommand{\mZ}{\mat{Z}} +\providecommand{\mGamma}{\mat{\Gamma}} +\providecommand{\mPi}{\mat{\Pi}} +\providecommand{\mSigma}{\mat{\Sigma}} +\providecommand{\mOmega}{\mat{\Omega}} +\providecommand{\mXi}{\mat{\Xi}} +\providecommand{\mDelta}{\mat{\Delta}} +\providecommand{\mLambda}{\mat{\Lambda}} +\providecommand{\mPhi}{\mat{\Phi}} + + +\providecommand{\va}{\vect{a}} +\providecommand{\vb}{\vect{b}} +\providecommand{\vc}{\vect{c}} +\providecommand{\vd}{\vect{d}} +\providecommand{\ve}{\vect{e}} +\providecommand{\vf}{\vect{f}} +\providecommand{\vE}{\vect{E}} +\providecommand{\vF}{\vect{F}} +\providecommand{\vg}{\vect{g}} +\providecommand{\vh}{\vect{h}} +\providecommand{\vk}{\vect{k}} +\providecommand{\vm}{\vect{m}} +\providecommand{\vn}{\vect{n}} +\providecommand{\vl}{\vect{l}} +\providecommand{\vp}{\vect{p}} +\providecommand{\vq}{\vect{q}} +\providecommand{\vr}{\vect{r}} +\providecommand{\vs}{\vect{s}} +\providecommand{\vt}{\vect{t}} +\providecommand{\vu}{\vect{u}} +\providecommand{\vw}{\vect{w}} +\providecommand{\vU}{\vect{U}} +\providecommand{\vx}{\vect{x}} +\providecommand{\vy}{\vect{y}} +\providecommand{\vz}{\vect{z}} +\providecommand{\vi}{\vect{i}} +\providecommand{\vj}{\vect{j}} +\providecommand{\vv}{\vect{v}} +\providecommand{\veta}{\vect{\eta}} +\providecommand{\vchi}{\vect{\chi}} + + +\providecommand{\vX}{\vect{X}} +\providecommand{\vY}{\vect{Y}} + + + + +\providecommand{\vgamma}{\vect{\gamma}} +\providecommand{\vdelta}{\vect{\delta}} +\providecommand{\vlambda}{\vect{\lambda}} +\providecommand{\vmu}{\vect{\mu}} +\providecommand{\vomega}{\vect{\omega}} +\providecommand{\vsigma}{\vect{\sigma}} +\providecommand{\vxi}{\vect{\xi}} +\providecommand{\valpha}{\vect{\alpha}} +\providecommand{\vphi}{\vect{\phi}} + +\providecommand{\vzero}{\vect{0}} +\providecommand{\vone}{\vect{1}} + +% symbols with wide hats +\providecommand{\whu}{\wh{u}} \providecommand{\whf}{\wh{f}} + +% symbols with wide tildas +\providecommand{\w}{\omega} + +\endinput diff --git a/patterns/main_patterns.tex b/patterns/main_patterns.tex new file mode 100644 index 0000000..b496197 --- /dev/null +++ b/patterns/main_patterns.tex @@ -0,0 +1,117 @@ +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% writeLaTeX Example: A quick guide to LaTeX +% +% Source: Dave Richeson (divisbyzero.com), Dickinson College +% +% A one-size-fits-all LaTeX cheat sheet. Kept to two pages, so it +% can be printed (double-sided) on one piece of paper +% +% Feel free to distribute this example, but please keep the referral +% to divisbyzero.com +% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% How to use writeLaTeX: +% +% You edit the source code here on the left, and the preview on the +% right shows you the result within a few seconds. +% +% Bookmark this page and share the URL with your co-authors. They can +% edit at the same time! +% +% You can upload figures, bibliographies, custom classes and +% styles using the files menu. +% +% If you're new to LaTeX, the wikibook is a great place to start: +% http://en.wikibooks.org/wiki/LaTeX +% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\documentclass[10pt,landscape]{article} +\usepackage{amssymb,amsmath,amsthm,amsfonts} +\usepackage{multicol,multirow} +\usepackage{calc} +\usepackage{ifthen} +%\usepackage[document]{ragged2e} +\usepackage{helvet} +\renewcommand{\familydefault}{\sfdefault} +\usepackage{wrapfig} +\usepackage[fontsize=8pt]{fontsize} + +\usepackage[landscape]{geometry} +\usepackage{echodefs} +\geometry{a4paper, landscape, margin=0.25cm} +\usepackage[colorlinks=true,citecolor=blue,linkcolor=blue]{hyperref} +\usepackage[ + protrusion=true, + activate={true,nocompatibility}, + final, + tracking=true, + kerning=true, + spacing=true, + factor=1100]{microtype} +\SetTracking{encoding={*}, shape=sc}{40} +%%Packages added by Sebastian Lenzlinger: +\usepackage{enumerate} %% Used to change the style of enumerations (see below). +\usepackage{tikz} %% Pagacke to create graphics (graphs, automata, etc.) +\usetikzlibrary{automata} %% Tikz library to draw automata +\usetikzlibrary{arrows} %% Tikz library for nicer arrow heads +%%End +%\microtypecontext{spacing=nonfrench} + +\ifthenelse{\lengthtest { \paperwidth = 11in}} + { \geometry{top=.5cm,left=.5cm,right=.5cm,bottom=.5cm} } + {\ifthenelse{ \lengthtest{ \paperwidth = 297mm}} + {\geometry{top=0.3cm,left=0.3cm,right=0.3cm,bottom=0.3cm} } + {\geometry{top=0.5cm,left=0.5cm,right=0.5cm,bottom=0.5cm} } + } +\pagestyle{empty} +\makeatletter +%% Renew default font +\usepackage{units} +\renewcommand{\section}{\@startsection{section}{1}{0mm}% + {0.1mm}% + {0.0001mm}%x + {\normalfont\normalsize\bfseries}} +\renewcommand{\subsection}{\@startsection{subsection}{2}{0mm}% + {0.0001mm}% + {0.00001mm}% + {\normalfont\small\bfseries}} +\renewcommand{\subsubsection}{\@startsection{subsubsection}{3}{0mm}% + {-1ex plus -.5ex minus -.2ex}% + {1ex plus .2ex}% + {\normalfont\small\bfseries}} +\makeatother +\setcounter{secnumdepth}{0} +\setlength{\parindent}{0pt} +\setlength{\parskip}{0pt plus 0.5ex} + +% ----------------------------------------------------------------------- + +\title{Pattern Recognition} + +\begin{document} +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Custom Commands +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\renewcommand{\l}[1]{\mathcal{L}(#1)} +\newcommand{\s}{\Sigma} +\newcommand{\then}{\rightsquigarrow} +\renewcommand{\empty}{\varnothing} +\newcommand{\any}{$\forall$} +\newcommand{\some}{$\exists$} +\newcommand{\predux}{$\leq_p$} +\newcommand{\tin}{$\in$} +\newcommand{\ntin}{$\not\in$} +\newcommand{\ffrom}[1]{\stackrel{(#1)}{\Rightarrow}} +\raggedright +\footnotesize +\microtypecontext{spacing=nonfrench} +\begin{multicols*}{4} +\setlength{\premulticols}{0.1cm} +\setlength{\postmulticols}{0.1cm} +\setlength{\multicolsep}{0.1cm} +\setlength{\columnsep}{0.1cm} +\input{patterns/topics/ingeredience} +\input{patterns/topics/knn} +\end{multicols*} +\end{document} \ No newline at end of file diff --git a/patterns/topics/decision.tex b/patterns/topics/decision.tex new file mode 100644 index 0000000..e69de29 diff --git a/patterns/topics/ingeredience.tex b/patterns/topics/ingeredience.tex new file mode 100644 index 0000000..383b07f --- /dev/null +++ b/patterns/topics/ingeredience.tex @@ -0,0 +1,33 @@ +\section{Intro} +\textbf{Data Space $\calX$}: Space of objects under investigation. \emph{Example} RGB images in $128 \times 128$ res with 8-bit color depth per channel, $\calX = \{0,1, \ldots, 255\}^{128 \times 128 \times 3};$\\ +\textbf{Label Space $\calY$:} Space of objects, kinds, types, attributes relevant to the pattern recognition problem. Examples: \emph{Classification:} Discrete Sets like $\calY = \{0,1\},\{-1,1\},\{jaszz,classic,\ldots\}$\\ +\emph{Regression:} Intervals like predicted temp $\calY = [-30, 50]$\\ +\textbf{Hypothesis Space $\calH$:} Space (set) of functions/ algorithms that take inputs from $\calX$ and yield outputs in $\calY$. +E.g. set of binary classifiers, set of $K$-NN classifiers, set of neural network classifiers.\\ +\textbf{Loss function $\ell$}: To chose algo / func from $\calH$, need measure of "best". $\ell: \calY \times \calY$: Given true label $y$ and estimate $\wh{y}$, $\ell(y \wh{y})$ measures how far they are from each other. \\ +\textbf{(Joint) probability distribution over data and labels $\PP$:} +We model the data--label pairs $(\vx, y)$ (elements in $\calX \times \calY$) as being (independent) samples from some probability distribution $\PP$ over $\calX \times \calY$. This probability distribution is a priori just a convenient way to mathematically formalize dealing with large amounts of data. It is an ephemeron. It does not exist, or at least it does not have to, in any concrete sense, for the theory and algorithmic machinery of pattern recognition to operate. + At this point, our goal is to find a function $h^*$ in $\calH$ which incurs the smallest average loss, with average being the expectation w.r.t. $\mathbb{P}$: + $ + \label{eq:risk-minimization} + h^* = \argmin_{h \in \calH} \EE_{(\vX, Y) \sim \PP}[ \ell(h(\vX), Y)], + $ + where in the subscript of the expectation, we emphasize that $\vX$ and $Y$ are jointly distributed random variables with distribution $\PP$. + \footnote{When we want to emphasize that something is a random variable, we will typeset it in capitals, as is standard. Thus $\PP[Y = y]$ is the probability that the random variable $Y$ assumes value $y$.} + We will use the name \textit{risk} for the average loss of $h$ and give it a nice letter : + $ + R(h) := \mathbb{E}[\ell(h(\vX), Y)]. + $ + This highlights a fundamental challenge in machine learning: the actual distribution of the probability distribution $\PP$ is \textit{unknown}. We thus introduce the last ingredient in supervised machine learning: its royal highness, the data. + + % + \footnote{The operation $\min_{h \in \calH} F(h)$ returns the smallest number in the set $\{ F(h) \mid h \in \calH \}$; the operation $\argmin_{h \in \calH}$ tells us for which $h$ this smallest value is achieved. For example, if simply $\calH = \mathbb{R}$ and $F(x) = x^2 - 5x - 2$, then $\min_{x \in \calH} F(x) = -\tfrac{33}{4}$ while $\argmin_{x \in \calH} = \tfrac{5}{2}$}.\\ + +\textbf{The (training) data set $\calD = \set{ (\vx_1, y_1), (\vx_2, y_2), \ldots, (\vx_N, y_N)}$:} The training data consists of pattern--label pairs drawn independently from $\PP$. Assumption: data are i.i.d. The training data set is a proxy to $\PP$---the larger it is the better. + +\subsection{Training and testing:} +Since we don't know $\PP$ we also cannot compute \eqref{eq:risk-minimization}. So instead we use \emph{empirical risk minimization}: +$ +\label{eq:empirical-risk-minimization} +TODO +$ diff --git a/patterns/topics/knn.tex b/patterns/topics/knn.tex new file mode 100644 index 0000000..f07552e --- /dev/null +++ b/patterns/topics/knn.tex @@ -0,0 +1,3 @@ +\textbf{KNN - Klassifier:} +\emph{training}: Is train data. +\emph{test}: Get distance e.g. $\norm{\mX - \vx}$.Chose $k$ closest. \ No newline at end of file diff --git a/patterns/topics/wiener.tex b/patterns/topics/wiener.tex new file mode 100644 index 0000000..e69de29 diff --git a/thoc/main.tex b/thoc/main.tex index 0b80f4d..59d632f 100644 --- a/thoc/main.tex +++ b/thoc/main.tex @@ -94,7 +94,7 @@ % ----------------------------------------------------------------------- -\title{Theory of Computer Science} +\title{Pattern Recognition} \begin{document} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -119,6 +119,7 @@ \setlength{\multicolsep}{0.1cm} \setlength{\columnsep}{0.1cm} + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Finite Automata %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%