diff --git a/trunk/RTCSA_SS/01Introduction.tex b/trunk/RTCSA_SS/01Introduction.tex index d0fd30d..da4967b 100644 --- a/trunk/RTCSA_SS/01Introduction.tex +++ b/trunk/RTCSA_SS/01Introduction.tex @@ -8,14 +8,17 @@ all days.\textcolor{red}{(source?)}. By the low resolution thermal camera, we can obtain the daily activities informations, but not reveal too much privacy like the RGB camera. -{\bf Contribution} Contribution +{\bf Contribution} In this work, we deployed multiple low resolution thermal camera to monitor the -turn over frequency while sleeping. With multiple thermal camera and -Super-resolution techniques we can improve the accuracy of turn over detection -by XX\%. +turn over frequency while sleeping. With multiple thermal camera, +Super-resolution techniques, and our method, we can have 50\% recall rate and 83\% precision +on turning over detection. -The remaining of this paper is organized as follow. Section~\ref{sec:bk_related} presents related works and background for developing the methods. Section~\ref{sec:design} presents the system architecture, and the developed mechanisms. Section~\ref{sec:eval} presents the evaluation results of proposed mechanism and Section~\ref{sec:conclusion} summaries our works. +The remaining of this paper is organized as follow. Section~\ref{sec:bk_related} +presents background for developing the methods. Section~\ref{sec:design} presents +the system architecture, and the developed mechanisms. Section~\ref{sec:eval} +presents the evaluation results of proposed mechanism and Section~\ref{sec:conclusion} summaries our works. - \ No newline at end of file + diff --git a/trunk/RTCSA_SS/02Background.tex b/trunk/RTCSA_SS/02Background.tex index 553b291..f066f09 100644 --- a/trunk/RTCSA_SS/02Background.tex +++ b/trunk/RTCSA_SS/02Background.tex @@ -1,9 +1,8 @@ -\section{Background and Related Works} +\section{Background} \label{sec:bk_related} This section describes background of Super-resolution technique, and the sensors -we are using in this work. The related works describes the works that using -thermal image to recognize activity. +we are using in this work. \subsection{Background} \paragraph{Super-Resolution Convolutional Neural Network} @@ -12,19 +11,27 @@ C. Dong et al.~\cite{ChaoDong16} proposed a deep learning method for single imag super-resolution (SR). Their method learns how to directly map the low-resolution image to high-resolution image. They show that the traditional sparse coding based SR methods can be reformulated into a deep convolutional neural network. -SRCNN consists three operations, illustrated in Figure~\ref{fig:SRCNN_model}. +SRCNN consists three operations.%, illustrated in Figure~\ref{fig:SRCNN_model}. The first layer of SRCNN model extracts the patches from low-resolution image. The second layer maps the patches from low-resolution to high-resolution. The third layer will reconstruct the high-resolution image by the high-resolution patches. -\begin{figure}[htb] - \begin{center} - \includegraphics[width=1\linewidth]{figures/SRCNN_model.pdf} - \caption{Illustration of super-resolution convolutional neural network (SRCNN) model~\cite{ChaoDong16}.} - \label{fig:SRCNN_model} - \end{center} -\end{figure} +%% \begin{figure}[htb] +%% \begin{center} +%% \includegraphics[width=1\linewidth]{figures/SRCNN_model.pdf} +%% \caption{Illustration of super-resolution convolutional neural network (SRCNN) model~\cite{ChaoDong16}.} +%% \label{fig:SRCNN_model} +%% \end{center} +%% \end{figure} + +\subsection{Thermal cameras} +In this work, we use two different resolution thermal camera to play the role +of low-resolution and high-resolution camera. For low-resolution camera, we use +Grid-EYE thermal camera. Grid-EYE is a thermal camera that can output +$8 \times 8$ pixels thermal data with $2.5^\circ C$ accuracy and $0.25^\circ C$ +resolution at $10$ fps. For the high-resolution one, we use Lepton 3. The +specification of them are shown in Table~\ref{table:specificationofdevices}. \begin{table}[hb] @@ -55,19 +62,11 @@ patches. Data Rate & 7.68 kbps & 2.34 mbps\\ \hline \end{tabular} - \caption{Specification of Grid-EYE and Lepton 3.} + \caption{Specification of Grid-EYE~\cite{grideye_datasheet} and Lepton 3~\cite{lepton_datasheet}.} \label{table:specificationofdevices} \end{table} -\subsection{Thermal cameras} -In this work, we use two different resolution thermal camera to play the role -of low-resolution and high-resolution camera. For low-resolution camera, we use -Grid-EYE thermal camera. Grid-EYE is a thermal camera that can output -$8 \times 8$ pixels thermal data with $2.5^\circ C$ accuracy and $0.25^\circ C$ -resolution at $10$ fps. For the high-resolution one, we use Lepton 3. The -specification of them are shown in Table~\ref{table:specificationofdevices}. - -\subsection{Related Works} +%\subsection{Related Works} %X. Chen et al.~\cite{7805509} proposed to use the visible camera as a guidance to super resolution the IR images. They proposed the IR-RBG multi-sensor imaging system. The approach bases on the fact that RGB channels have different correlation with IR images. To avoid wrong texture transfer, the method used cross correlation to find the proper matching between the region of IR image and RGB image. Next, the method applied sub-pixel estimation and guided filtering to remove the noise and discontinuities in IR image. At the last step of the iteration, they used truncated quadric model as the cost function to terminate the algorithm. After the iteration, the method fine tuned the output IR image with outlier detection to remove the black points which only appear at the edge of IR image. % \begin{figure}[ht] % \begin{center} @@ -88,4 +87,4 @@ specification of them are shown in Table~\ref{table:specificationofdevices}. % \end{figure} % % background -The assumption on the resolution of thermal images in above works is at least $120 \times 160$. Compared to the $8 \times 8$ thermal images collected by Grid-EYE sensors, more thermal features are available. In addition, these works use RGB images as reference to train the model. In this work, RGB camera will not be used so as to protect privacy. +%The assumption on the resolution of thermal images in above works is at least $120 \times 160$. Compared to the $8 \times 8$ thermal images collected by Grid-EYE sensors, more thermal features are available. In addition, these works use RGB images as reference to train the model. In this work, RGB camera will not be used so as to protect privacy. diff --git a/trunk/RTCSA_SS/03Design.tex b/trunk/RTCSA_SS/03Design.tex index 1660667..4277659 100644 --- a/trunk/RTCSA_SS/03Design.tex +++ b/trunk/RTCSA_SS/03Design.tex @@ -3,15 +3,26 @@ \subsection{System Architecture} We designed a thermal-box to collect the data. It has four Grideye sensors on the -corners of a 10 cm square and a Lepton 3 at the central. Our method is made by -three parts. The first part is to train the SRCNN model with fused Grideye image -as low-resolution and downscaled Lepton 3 image. The second part, we use the -Super-resolution image to train a neural network model to recognize current pose -is lay on back or lay on side. The third part, because of noise and the residual +corners of a 10 cm square and a Lepton 3 at the central. Figure~\ref{fig:method} shows +the system of our method. It consists four parts. The first part is to fuse multiple +Grideye image, since the resolution of singel Grideye sensor only has 64 pixels. The +second part, we train the SRCNN model with fused Grideye image +as low-resolution and downscaled Lepton 3 image as high-resolution image. +The third part, we use the +Super-resolution image to train a neural network model for recognizing current pose +is lay on back or lay on side. The last part, because of noise and the residual heat on bed after turn over, it is difficult to figure out the current pose. We remove the noise by median filter, and determine the current pose according to the trend of the possibility from recognition network. +\begin{figure}[htb] + \begin{center} + \includegraphics[width=1\linewidth]{figures/method.pdf} + \caption{Illustration of Proposed Method.} + \label{fig:method} + \end{center} +\end{figure} + \subsection{Grideye Data Fusion} On the thermal-box, there are four Grideye sensors. At the beginning, we let diff --git a/trunk/RTCSA_SS/04Evaluation.tex b/trunk/RTCSA_SS/04Evaluation.tex index f6b2983..f126101 100644 --- a/trunk/RTCSA_SS/04Evaluation.tex +++ b/trunk/RTCSA_SS/04Evaluation.tex @@ -19,7 +19,7 @@ Lepton 3, and align them at the same timestamps. Figure~\ref{fig:resolution_comp \subfloat[Downscaled Lepton Image]{ \includegraphics[width=0.32\columnwidth]{figures/HR.png} } - \caption{Result of SRCNN} + \caption{Result of SRCNN.} \label{fig:resolution_compare} \end{figure} @@ -28,4 +28,6 @@ of lay on right or left side. The result shows that the accuracy of single frame can be improved about 5\% by SRCNN. We let a person lay on bed and change his pose every minutes. The pose is repeating -lay on back, lay on left, lay on back and lay on right. +lay on back, lay on left, lay on back and lay on right. Our method will output the currect pose +every 10 seconds and check if the pose changed. The accuracy of pose detection is 65\%, and the +turning over detection has 50\% recall rate and 83\% precision. diff --git a/trunk/RTCSA_SS/Main.tex b/trunk/RTCSA_SS/Main.tex index 48c8837..0c6069c 100644 --- a/trunk/RTCSA_SS/Main.tex +++ b/trunk/RTCSA_SS/Main.tex @@ -1,6 +1,6 @@ % Final Submission -%\documentclass[10pt,conference,a4paper,twocolumn]{./IEEEtran} -\documentclass[10pt,a4paper,twocolumn]{ieeeconf} +\documentclass[10pt,conference,a4paper,twocolumn]{./IEEEtran} +%\documentclass[10pt,a4paper,twocolumn]{ieeeconf} % Draft %\documentclass[10pt,a4paper,onecolumn]{ieeeconf} %\documentclass[12pt,a4paper,onecolumn]{IEEEtran} @@ -76,6 +76,9 @@ %%%%%%%%%%%%%%%%% %\input{MySetting} +\title{Turning Over Detecting Using Low-Resolution Thermal Sensor} + +\author{Jyun-Jhe Chou} %\author{ % \IEEEauthorblockN{Jyun-Jhe \textsc{Chou}\IEEEauthorrefmark{1}, Chi-Sheng \textsc{Shih}\IEEEauthorrefmark{1}, Wei-Dean \textsc{Wang}\IEEEauthorrefmark{2}, and Kuo-Chin \textsc{Huang}\IEEEauthorrefmark{3}} % \IEEEauthorblockA{\IEEEauthorrefmark{1}\small{Embedded Systems and Wireless Networking Lab}\\ @@ -87,13 +90,14 @@ % \IEEEauthorblockA{\IEEEauthorrefmark{2}Department of Medical Education and Bioethics,\\NTU} % \IEEEauthorblockA{\IEEEauthorrefmark{3}Department of Family Medicine, \\NTU Hospital} %} -\author{Jyun-Jhe Chou} -\affil{\small{Graduate Institute of Networking and Multimedia}\\ - \small{Department of Computer Science and Information Engineering}\\ - \small{NTU IoX Research Center}\\ - \small{National Taiwan University} \vspace{1.5ex}} +\author{Jyun-Jhe \textsc{Chou}\IEEEauthorrefmark{1}, and Chi-Sheng \textsc{Shih}\IEEEauthorrefmark{2}\\ + \small{Embedded Systems and Wireless Networking Lab}\\ + \small{\IEEEauthorrefmark{2}Graduate Institute of Networking and Multimedia}\\ + \small{Department of Computer Science and Information Engineering}\\ + \small{National Taiwan University}\\ + \small{Email: \IEEEauthorrefmark{1}{\tt f03922104@csie.ntu.edu.tw}, \IEEEauthorrefmark{2}{\tt cshih@csie.ntu.edu.tw}} +} -\title{Title} % \setlength{\topmargin}{-0.5in} %\setlength{\textheight}{10.1in} diff --git a/trunk/RTCSA_SS/Revisions/RTCSA20_SS_20200603.pdf b/trunk/RTCSA_SS/Revisions/RTCSA20_SS_20200603.pdf new file mode 100644 index 0000000..859f01f Binary files /dev/null and b/trunk/RTCSA_SS/Revisions/RTCSA20_SS_20200603.pdf differ diff --git a/trunk/RTCSA_SS/bibliography.bib b/trunk/RTCSA_SS/bibliography.bib index e149e72..98f12e7 100644 --- a/trunk/RTCSA_SS/bibliography.bib +++ b/trunk/RTCSA_SS/bibliography.bib @@ -24,8 +24,8 @@ Cyber-Physical Systems}, month={Feb},} @misc{lepton_datasheet, - title = {FLIR LEPTON® 3 Long Wave Infrared (LWIR) Datasheet}, - howpublished = {\url{https://media.digikey.com/pdf/Data\%20Sheets/FLIR\%20PDFs/Lepton3_LWIR_DS_Rev100_4-4-17.pdf}}, + title = {FLIR LEPTON\textregistered 3 Long Wave Infrared (LWIR) Datasheet}, + howpublished = {https://groupgets-files.s3.amazonaws.com/lepton/Lepton-3-Engineering-Datasheet.pdf}, } @misc{grideye_datasheet, diff --git a/trunk/RTCSA_SS/figures/method.pdf b/trunk/RTCSA_SS/figures/method.pdf new file mode 100644 index 0000000..2d2c20f Binary files /dev/null and b/trunk/RTCSA_SS/figures/method.pdf differ