\documentclass{beamer}
\usepackage{url}
\usepackage{algorithm2e}
\usepackage[MeX]{polski}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{tikz}
\usetikzlibrary{arrows}
\usetheme{Warsaw}
\usepackage{color}
\usepackage[utf8x]{inputenc}
\usepackage{default}
\usepackage{epstopdf}

\title{HMM for Bioinformatics}
\author{\textbf{Pawe{\l} B{\l}a{\.z}ej}\\
Department of Genomics, Faculty of Biotechnology,\\
blazej@smorfland.uni.wroc.pl}

\begin{document}
\begin{frame}
\titlepage
\end{frame}
\begin{frame}
  \frametitle{Conventions and notations}
 \begin{enumerate}
 \item <1-> ${\cal X}, {\cal Y}$ -- alphabet;
 \item <2-> $|{\cal X}|=|{\cal Y}|=L$ -- size of an alphabet;  
 \item <3-> $X,Y$ discrete random variables ($X:\ \Omega\to {\cal X}$);
 \item <4->  $P_{X}(x_i)$ probability.  
\end{enumerate}    
\end{frame}

\begin{frame}
\frametitle{Entropy}  
We can think of entropy as the level of uncertainty associated with a random variable.
\end{frame}  

\begin{frame}
\frametitle{Entropy -- Definition}
We define the entropy of the random variable $X\sim P$ as:
$$
H(P)=H(X)=-\sum_{i=1}^{L}P(x_i)log_2(P(x_i))
$$
where $H(0)=0$ (convention $0\cdot log_20=0$).
\end{frame}  

\begin{frame}
\frametitle{Joint entropy}
$$
H(X,Y)=-\sum_{x}\sum_{y}P(x,y)logP(x,y)
$$

\end{frame}
\begin{frame}
\frametitle{conditional entropy}
$$
H(X|Y)=-\sum_{x}\sum_{y}P(x,y)logP(x|y).
$$
\end{frame}  


\begin{frame}
\frametitle{Properties of the entropy function}
\begin{enumerate}
\item <1-> $H(X)\geq 0$  
\item <2-> $H(X)\leq log(L)$ with equality iff $X$ is uniformly distributed i.e. $P(x_i)=1/L$ for every $i$;
\item <3->  $H(X,Y)\leq H(X)+H(Y)$ with equality iff $X$ and $Y$ are independent;
\item <4-> $H(X)$ is concave in $X$ . 
\end{enumerate}
\end{frame}  
\begin{frame}
\frametitle{Chain Rule for entropy}
$$
H(X,Y)=H(X)+H(Y|X)=H(Y)+H(X|Y).
$$
\end{frame}  

\begin{frame}
  \frametitle{mutual information}
  $$
  M(X,Y)=\sum_{x}\sum_{y}P(x,y)log\frac{P(x,y)}{P(x)P(y)}
  $$
\pause  
\begin{block}{Remark}
  $$
  M(X,Y)= H(H)+H(Y)-H(X,Y).
$$
\end{block}  
\end{frame}  

\begin{frame}
\frametitle{Example -- Binary entropy function}
${\cal X}=\{x_1,x_2\}$, $P(x_1)=p,\ P(x_2)=1-p$
$$
H(p)=-plog_2p-(1-p)log_2(1-p)
$$
\end{frame}
\begin{frame}
  \frametitle{Example -- Entropy of random DNA}
  \begin{enumerate}
  \item <1-> ${\cal X}=\{A,T,G,C\}$;
  \item <2-> $p(X=A)=p(X=T)=\ldots =p(X=C)=0.25$;
  \item <3-> $H(X)=(\frac{1}{4}log\frac{1}{4}+\ldots +\frac{1}{4}log\frac{1}{4})=2$;
 \item <4-> We can think in this case of the entropy as a number of yes/no questions needed to identify an autcome.   
  \end{enumerate}
\end{frame}
  \begin{frame}
    \frametitle{Entropy -- Interpretation}
    \begin{block}{Definition}
      A code is called uniquely decodable if any string composed of finite number of symbols from ${\cal X}$ gets a unique codestring.
    \end{block}  
    \pause
    \begin{block}{Kraft inequality}
      A necessary and sufficient condition on the lengths of codestring  $l_i$ in a uniquely decodable code is:
      $$
      \sum_{i=1}^{L} 2^{-l_i}\leq 1.
      $$
    \end{block}
    
    
  \end{frame}

  \begin{frame}
        \frametitle{Entropy -- Interpretation}
The expected length of the optimal uniquely decodable code belongs to the interval $[H(X), H(X)+1)$
    \end{frame}  

 \begin{frame} 
\frametitle{relative entropy -- Kullback distance}
$$
D(P_1|P_2)=\sum_{i=1}^{L}P_1(x_i)log\frac{P_1(x_i)}{P_2(x_i)}
$$
\pause
\begin{block}{Convention}
\begin{enumerate}
\item  $0\cdot log\frac{0}{P_2(x_i)}=0$;
\item  $P_2(x_i)log\frac{P_1(x_i)}{0}=\infty$
\end{enumerate}
\end{block}  
\end{frame}  

\begin{frame}
  \frametitle{Information content}
  If X is a random variable with the probability $P_1$ on an alphabet of $L$ symbols, and $P_2$ is the uniform distribution on this alphabet then
$$
D(P_1|P_2) =log(L)-H(X).
$$
\end{frame}  

\begin{frame}
\frametitle{Properties of relative entropy}
\end{frame}

\begin{frame}
  \frametitle{Further reading}
\begin{enumerate}
\item T. Inglot, ,,Teoria informacji a statystyka matematyczne'' wykład wygłoszony na XXXVIII Konferencji Statystyka Matematyczna Wisła 2012;
\item L. Dębowski, ''Information Theory and Statistics'', Institute of
  Computer Science Polish Academy of Science, Warsaw, 2013.  

\end{enumerate}  
\end{frame}

\end{document}




%%% Local Variables:
%%% mode: latees of the ex
%%% TeX-master: t
%%% End:
