\documentclass[11pt]{article}
\usepackage{graphicx}
\usepackage{times}
\usepackage{amssymb}
\usepackage{float}
\usepackage{amsmath,amssymb,amsfonts,bm}
\newcount\refno\refno=1
\def\ref{\the\refno \global\advance\refno by 1}
\def\ux{\underline{x}}
\def\uw{\underline{w}}
\def\ut{\underline{\theta}}
\def\umu{\underline{\mu}}
\def\be{p_e^*}
\newcount\eqnumber\eqnumber=1
\def\eq{\the \eqnumber \global\advance\eqnumber by 1}
\def\eqs{\eq}
\def\eqn{\eqno(\eq)}
\pagestyle{empty}
\def\baselinestretch{1.1}
\topmargin1in \headsep0.3in
\topmargin0in \oddsidemargin0in \textwidth6.5in \textheight8.5in
\begin{document}
\setlength{\parskip}{1.2ex plus0.3ex minus 0.3ex}
\thispagestyle{empty} \pagestyle{myheadings} \markright{Homework
1: CS 274A, Probabilistic Learning: Winter 2018}
\title{CS 274A Homework 1}
\author{Probabilistic Learning: Theory and Algorithms, CS 274A, Winter 2018}
\date{Due Date: Wednesday January 17th, submit hardcopy at the start of class}
\maketitle
\section*{Instructions and Guidelines for Homeworks}
\begin{itemize}
\item
Please answer all of the questions and submit a {\bf hardcopy} of your written solutions
{\bf at the start of class on the due date}
(either hand-written or typed are fine as long as the writing is legible). Clearly mark
your name on the first page.
%Code (if requested) should be submitted to the EEE dropbox. No
%need to submit any code unless we request it.
\item
All problems are worth 10 points unless otherwise stated. All homeworks will get equal weight in computation of the final grade for the class.
\item
The homeworks are intended to help you work through the concepts
we discuss in class in more detail. It is important that you try
to solve the problems yourself. The homework problems are important to help you better
learn and reinforce the material from class. If you don't
do the homeworks you will likely have difficulty in the exams
later in the quarter.
\item If you can't solve a
problem, you can discuss it {\it verbally} with another student. However, please note that before you submit your homework solutions you
are not allowed to view (or show to any other student) any {\it written material} directly related to the homeworks, including other students' solutions or drafts of solutions, solutions from previous versions of this class, and so forth. The work you hand in should be your own original work.
\item You are allowed to use reference materials in your solutions, such as class notes, textbooks, other reference material (e.g., from the Web), or solutions to other problems in the homework. It is strongly recommended that you first try to solve the problem yourself, without resorting to looking up solutions elsewhere. If you base your solution on material that we did not discuss in class, or is not in the class notes, then you need to clearly provide a reference, e.g., ``based on material in Section 2.2 in ....."
\item
In problems that ask for a proof you should submit a complete mathematical
proof (i.e., each line must follow logically from the preceding one, without
``hand-waving"). Be as clear as possible in explaining
your notation and in stating your reasoning as you go from line to line.
\item
If you wish to use LaTeX to write up
your solutions you may find it useful to use the .tex file for this homework
that is posted on the Web page.
\end{itemize}
\vfill\eject
If you need to brush up on your knowledge of probability, reading Note Sets 1 and 2 from the class Web page is recommended before attempting the problems below.
\subsection*{Problem \ref: (Linearity of Expectation)}
The expected value of a continuous random variable $X$, taking values $x$, is defined as $\mu_x = E[X] = \int p(x) \ x \ dx$ where $p(x)$ is the probability density function for $X$. The variance is defined as $var(X) = E[ (X - \mu_x)^2] = \int p(x) (x - \mu_x)^2 dx$ (often also denoted as $\sigma^2_x$).
\begin{enumerate}
\item Prove that expectation is linear, i.e., that $E[aX + b] = aE[X] + b$ where $a$ and $b$ are constants.
\item Prove that $var(cX) = c^2 var(X)$ where $c$ is a constant.
\item Prove that $ var(X) = E[X^2] - (E[X])^2$.
\end{enumerate}
\subsection*{Problem \ref: (Uniform Density)}
Let $X$ be a continuous random variable with uniform density $U(a,b)$, with $a**1$.
\begin{enumerate}
\item Use the law of total probability and the first-order Markov property to derive an
efficient way to compute $P(X_{i+m} | x_i,\ldots,x_1)$.
\item What is the time complexity
of this computation as a function of $m$ and $K$? (in "big O" notation)?
\end{enumerate}
In answering this problem you can assume that the Markov chain is {\it homogeneous}, i.e., that
the transition probabilities $P(X_i | X_{i-1} )$ are the same for all values of $i$.
\subsection*{Problem \ref: (Naive Bayes Classification Model)}
The naive Bayes model is a probability model with a class variable $C$ taking $M$ possible values $c \in \{1,\ldots,M\}$ and $d$ features $X_1,\ldots, X_d$. For simplicity
we will assume that each of the $X_j$ variables are discrete and
each takes $K$ possible values $x_j \in \{1,\ldots,K\}$. Each feature
is conditionally independent of all the other features given $C$.
\begin{enumerate}
\item Write down the correct expression for the joint distribution $P(C, X_1,\ldots,X_d)$ for this model.
\item Draw a picture of the graphical model for the case of $d=3$.
\item Specify exactly how many parameters are needed for
this model in the genereal case, as a function of $M, K,$ and $d$. A {\it parameter} in this context is
any probability or conditional probability value that is needed to specify the model.
\end{enumerate}
\subsection*{Problem \ref: (Graphical Model 1)}
Consider a directed graphical model with random variables $A, B, C, D, E, F$ where $F$ has parent $E$, $E$ has parents $C$ and $B$, $D$ has parent $C$, $C$ has parents $A$ and $B$, and $A$ and $B$ each have no parents. Assume that each variable can take $K$ values, $K \ge 2$.
\begin{enumerate}
\item Draw a diagram showing the structure of this graphical model and write down an expression for the joint distribution $P(a, b, c, d, e, f)$ as represented by this graphical model.
\item Specify precisely how many parameters (probabilities) are needed to specify this model. A parameter is defined (for this problem) as a conditional probability or a marginal (unconditional) probability.
\item How many parameters would be required if we had a saturated model? (i.e., a model with no conditional independencies assumed).
\end{enumerate}
\subsection*{Problem \ref: (Graphical Model 2)}
Consider another directed graphical model with discrete random variables $X, Y, V, Z$ where $X$ has no parents, $Y$ and $V$ each have $X$ as a parent, and $Z$ has $Y$ as a parent. You can assume that each variable takes $K$ values, $K \ge 2$.
\begin{enumerate}
\item Suppose a value $x$ is observed for $X$ and we don't know the values of any of the other variables. Given that $x$ is known, show how one would use the structure of the graphical model to compute $P(z|x)$ for any value $z$ of the variable $Z$. In particular, prove that this computation does not depend in any way on the conditional probability table $P(V|X)$.
\item Now say that both $x$ and $v$ (some value for $V$) are observed. How does this change the answer to part 1?
\item Now say $v$ is observed but $x$ is not. Show how you could compute $P(z|v)$ in a step by step manner by first computing $P(x|v)$, then computing $P(y|v)$, and finally computing $P(z|v)$, at each stage using the information from the previous step.
\end{enumerate}
\end{document}
**