try ipytho#5 , numpy, array,matrix, vector

//=== numpy support matrix operation by numpy.array
import numpy as np
arr= np.random.rand(3,5)

arr.shape             ### give the dimension of the matrix
Out[80]: (3L, 5L)

print arr
[[ 0.67248573  0.48092984  0.69184439  0.71945406  0.57882278]
 [ 0.79634702  0.02227532  0.77490943  0.36783077  0.98777438]
 [ 0.36116686  0.74907581  0.23774053  0.82991951  0.29351733]]

b=arr.T          ### perform matrix transpose

b
Out[83]:
array([[ 0.67248573,  0.79634702,  0.36116686],
       [ 0.48092984,  0.02227532,  0.74907581],
       [ 0.69184439,  0.77490943,  0.23774053],
       [ 0.71945406,  0.36783077,  0.82991951],
       [ 0.57882278,  0.98777438,  0.29351733]])

b.shape
Out[84]: (5L, 3L)

//=== indexer, i:j , i:j:s

i : j == i : j : 1  --> from i to j with increment step 1 ,  j is not included

i : j : s  --> from i to j with increment step s,  j is not included

 arr[ : , 0]  : get the column 0

 arr[..., :4] == arr[..., 0:4] : get column 0 -- 3 

 arr[..., :5] : get column 0 -- 4 

arr[:, 3:1:-1]

Out[95]: 

array([[ 0.71945406,  0.69184439],

       [ 0.36783077,  0.77490943],

       [ 0.82991951,  0.23774053]])

//=== bracket affect [ ]
arr= np.array([np.random.rand(3,5)])
arr.shape
Out[76]: (1L, 3L, 5L)

arr= np.array(np.random.rand(3,5))
arr.shape
Out[78]: (3L, 5L)

arr= np.random.rand(3,5)
arr.shape
Out[80]: (3L, 5L)

//=== double bracket affect
import numpy as np

arr1= np.array([1,2,3])

type(arr1)
Out[45]: numpy.ndarray

arr1.shape
Out[47]: (3L,)

arr2= np.array([[1,2,3]])
arr2.shape
Out[49]: (1L, 3L)

arr3= np.array([[1],[2],[3]])
arr3.shape
Out[51]: (3L, 1L)

arr4=arr3.T
arr4.shape
Out[53]: (1L, 3L)

arr5= arr1.T
arr5.shape
Out[55]: (3L,)

*** although type(arr1) shows numpy.ndarray ,
arr1 is actually a vector( === column vector === row vector ?)

the transpose of arr1 is still a vector containing 3 elements --> 3-tuple vector

i.e. transpose has no effect on vector

*** arr2 is a row 'vector' (single-row matrix)

*** arr3 is a column 'vector'(single-column matrix)

//=== the built-in list of Python not support transpose

L1=[1,2,3]

type(L1)

Out[61]: list

L1.T

---------------------------------------------------------------------------

AttributeError                            Traceback (most recent call last)

<ipython-input-62-167a2304fdf0> in <module>()

----> 1 L1.T

AttributeError: 'list' object has no attribute 'T'

LL1=[[1,2,3],[4,5,6]]

type(LL1)

Out[64]: list

LL1.T

---------------------------------------------------------------------------

AttributeError                            Traceback (most recent call last)

<ipython-input-65-49e25d799db3> in <module>()

----> 1 LL1.T

AttributeError: 'list' object has no attribute 'T'

try ipython#4, animation

//=== matplotlib, animation , ffmpeg
http://nbviewer.ipython.org/url/jakevdp.github.io/downloads/notebooks/AnimationEmbedding.ipynb
http://jakevdp.github.io/blog/2012/08/18/matplotlib-animation-tutorial/

""" ...
C:\Users\bob\Anaconda\lib\site-packages\matplotlib\animation.py:727: UserWarning: MovieWriter ffmpeg unavailable
  warnings.warn("MovieWriter %s unavailable" % writer)
... """

*** need ffmpeg to create mp4 file

https://www.ffmpeg.org/download.html#build-windows
--> http://ffmpeg.zeranoe.com/builds/

-->
dnload & install 7zip
dnload & extract ffmpeg
add ffmpeg/bin  to $Path

[
//=== ffmpeg build for windows
https://www.ffmpeg.org/download.html#build-windows
--> http://ffmpeg.zeranoe.com/builds/


//=== the latest archive of static-build ffmpeg for 32bit linux
http://ffmpeg.gusari.org/static/32bit/ffmpeg.static.32bit.latest.tar.gz
]

-->
""" ... Open the command prompt. Enter the command “ffmpeg –version”
If you receive a “libstdc++ -6 is missing” error, you may need to install the Microsoft Visual C++ Redistributable Package, which is available for free from Microsoft.
..."""


--> error
matplotlib\backends\backend_agg.pyc in print_raw(self, filename_or_obj, *args, **kwargs)
    503             close = False
    504         try:
--> 505             renderer._renderer.write_rgba(filename_or_obj)
    506         finally:
    507             if close:

RuntimeError: Error writing to file

--> has probolem with tempfile.NamedTemporaryFile()
use a local file under cwd instead, say , 'myani1.mp4'



//===
https://jakevdp.github.io/blog/2012/08/18/matplotlib-animation-tutorial/
http://matplotlib.org/1.4.2/examples/animation/index.html
http://matplotlib.org/1.4.2/api/animation_api.html


//===
import matplotlib.animation as animation
animation.Animation._repr_html_ = anim_to_html

animation.FuncAnimation
animation.ArtistAnimation
animation.TimedAnimation



//=== class matplotlib.animation.FuncAnimation(fig, func, frames=None, init_func=None, fargs=None, save_count=None, **kwargs)

""" ...
Bases: matplotlib.animation.TimedAnimation

Makes an animation by repeatedly calling a function func, passing in (optional) arguments in fargs.

frames can be a generator, an iterable, or a number of frames.

init_func is a function used to draw a clear frame. If not given, the results of drawing from the first item in the frames sequence will be used. This function will be called once before the first frame.

If blit=True, func and init_func should return an iterable of drawables to clear.

kwargs include repeat, repeat_delay, and interval: interval draws a new frame every interval milliseconds. repeat controls whether the animation should repeat when the sequence of frames is completed. repeat_delay optionally adds a delay in milliseconds before repeating the animation.
... """

//=== class matplotlib.animation.TimedAnimation(fig, interval=200, repeat_delay=None, repeat=True, event_source=None, *args, **kwargs)



//=== matplotlib.animation.FuncAnimation(fig, func, frames=None, init_func=None, fargs=None, save_count=None, **kwargs)

*** frames can be

•    a generator, 
•    an iterable, or 
•    a number of frames.

--> data_gen, 25, frames=600

def data_gen():
    t = data_gen.t
    cnt = 0
    while cnt < 1000:
        cnt+=1
        t += 0.05
        yield t, np.sin(2*np.pi*t) * np.exp(-t/10.)
data_gen.t = 0

def run(data):
    # update the data
    t,y = data
    xdata.append(t)
    ydata.append(y)
...
animation.FuncAnimation(fig, run, data_gen, blit=True, interval=10,   repeat=False)



def update_line(num, data, line):
    line.set_data(data[...,:num])
    return line,

line_ani = animation.FuncAnimation(fig1, update_line, 25, fargs=(data, l),  interval=50, blit=True)




def animate(i):
    x = np.linspace(0, 2, 1000)
    y = np.sin(2 * np.pi * (x - 0.01 * i))
    line.set_data(x, y)
    return line,
ani = animation.FuncAnimation(fig, animate, frames=600,
                              interval=10, blit=True, init_func=init)



//=== class matplotlib.animation.ArtistAnimation(fig, artists, *args, **kwargs)

???
""" ...
Bases: matplotlib.animation.TimedAnimation

Before calling this function, all plotting should have taken place and the relevant artists saved.

frame_info is a list, with each list entry a collection of artists that represent what needs to be enabled on each frame. These will be disabled for other frames.
... """

//===  matplotlib.animation.ArtistAnimation(fig, artists, *args, **kwargs)
ims = []
for add in np.arange(15):
    ims.append((plt.pcolor(x, y, base + add, norm=plt.Normalize(0, 30)),))

im_ani = animation.ArtistAnimation(fig2, ims, interval=50, repeat_delay=3000,
    blit=True)

http://matplotlib.org/1.4.2/api/artist_api.html

try ipython#3 , short cut

[Q] how to run ipython notebook cells at one time?
[Q] how to run a whole ipython notebook instead of an indivisual cell?

-->
https://github.com/ipython/ipython/pull/4319
https://github.com/ipython/ipython/issues/2228

* menu:  Cell -> Run All
* keyboard: CTRL M, R shortcut for "run all cells" in the notebook


//=== interrupt
menu: Kernel  -> Interrupt
or
keyboard: Ctrl-m i

//=== restart
menu: Kernel  -> Restart menu
or
keyboard: Ctrl-m .


//===
'dd' to delete a cell

ssh, add key to authorized_keys on ubuntu

clientA, key pair (privkeyA, pubkeyA), ip address ipA
clientB, ipB, key pair to be added
serverC (where sshd is running) ,

assume clientA can  already connect to bob@serverC  via ssh
i.e.
/home/bob/.ssh/authorized_keys already contains pubkeyA
/etc/ssh/sshd_config contains "AllowUsers bob@ipA"

1. on clientB,  create key pair (privkeyB, pubkeyB) via Git Bash
  $ ssh-keygen

2. cp pubkeyB from clientB to clientA via clipboard/email/ftp/...
  pubkeyB is like
  "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDc ... ... "
  * take care not to wrap!

3. on clientA, ssh to connect to serverC via putty
   [session: Data/Auto-login username, SSH/Auth]

* add pubkeyB to  authorized_keys
bob@serverC$ nano .ssh/authorized_keys

* add ipB to sshd_config
bob@serverC$ sudo nano /etc/ssh/sshd_config

* reload sshd config
bob@serverC$ sudo service ssh  reload

4. on clientB, ssh to connect to serverC with privkeyB
[for putty, has to convert privkeyB to .ppk format by puttyGen]

try ipython #2, notebook

//=== http://www.randalolson.com/2012/05/12/a-short-demo-on-how-to-use-ipython-notebook-as-a-research-notebook/

IPython Notebook : seamless integration of text and code



//=== cell types

Markdown cell
Heading cell
Code cell
...


//===
[Q]  how to start ipython notebook from ipython console?

In [1]: notebook
UsageError: too few arguments
In [2]: !ipython notebook
--> dangerous??



[Q] how to stop ipython notebook?
[Q] how to close IPython Notebook properly?

after closing chrome browser,
the kernel is shutdown automatically,
but then
ctl+c does not work , nor other keys function,
the whole terminal console is frozen!
at last
--> kill  python process from Task Manager

//===
[Q] how to start ipython notebook with different profile ?
--> ipython notebook --profile=3h
xxx ipython --profile=3h notebook



[Q] how to view the plot result in noteboook?

http://nbviewer.ipython.org/github/ipython/ipython/blob/1.x/examples/notebooks/Part%203%20-%20Plotting%20with%20Matplotlib.ipynb

%matplotlib inline
--> use the inline backend, which will embed plots inside the Notebook:


//=== http://ipython.org/ipython-doc/1/interactive/nbconvert.html
The command-line syntax to run the nbconvert script is:

> ipython nbconvert --to FORMAT notebook.ipynb
This will convert the IPython document file notebook.ipynb into the output format given by the FORMAT string.

* default format is html
> ipython nbconvert ex1.ipynb
> ipython nbconvert --to markdown ex2.pynb --stdout


a configuration file, say mycfg.py,
"""...
c = get_config()
c.NbConvertApp.notebooks = ["ex1.ipynb", "ex2.ipynb"]
..."""
and using the command:

> ipython nbconvert --config mycfg.py



//=== [Q] how to host ipython on internet?
http://nbviewer.ipython.org/github/minrk/ipython/blob/master/examples/Notebook/Running%20the%20Notebook%20Server.ipynb

""" ...
 Most of the time, users run the notebook server on their local computer using IPython's command line interface.
...
You can also specify the port manually:
> ipython notebook --port 9999

Or start notebook server without opening a web browser.
> ipython notebook --no-browser

... """

//=== Securing the notebook server

* Setting a password
* Encrypt network traffic using SSL

In [1]:
from IPython.lib import passwd
password = passwd("my secret pswd")
password
Out[1]:
'sha1:6c2164fc2b22:ed55ecf07fc0f985ab46561483c0e888e8964ae6'

add the hashed passwd to myconfig.py:

"""
# Password to use for web authentication
c = get_config()
c.NotebookApp.password = u'sha1:6c2164fc2b22:ed55ecf07fc0f985ab46561483c0e888e8964ae6'
"""



//=== certificate/private key for  ssl/https
adding the following to  myconfig.py:

"""
# The full path to an SSL/TLS certificate file.
c.NotebookApp.certfile = u'/Users/bgranger/.ipython/profile_my_profile/mycert.crt'

# The full path to a private key file for usage with SSL/TLS.
c.NotebookApp.keyfile = u'/Users/bgranger/.ipython/profile_my_profile/mycert.key'
"""


//=== make notebook server public
by listening to arbitrary ip address ?

> ipython notebook --ip=* --no-browser

or add the following to  myconfig.py:

"""
c.NotebookApp.ip = '*'
c.NotebookApp.open_browser = False
"""

try ipython #1

[Q] what's ipython ?
* an enhanced interactive Python interpreter, offering tab completion, object introspection...
* an enhanced command shell for Python
* high performance tools for parallel computing ?
* supporting interactive computing in any language ?

http://ipython.org/


//=== my environment, Win7x64

0. download and install Anaconda (Python+ iPython + notebook +  ... )
    http://continuum.io/downloads

1. open Anaconda cmd prompt


//=== install ipython by Anaconda
> conda update conda
> conda update ipython


or
traditional way
> pip install ipython

or include the dependencies for the IPython notebook:
> pip install "ipython[notebook]"

or
manually download IPython from Github or PyPI. To install one of these versions, unpack it and run the following from the top-level source directory using the Terminal:
> python setup.py install

* ipython is a module for python from this perspective


//=== how to start ipython
anaconda cmd prompt
> ipython



//=== http://ipython.org/ipython-doc/stable/interactive/tutorial.html
line-oriented and cell-oriented magics.

""" ...
Line magics are prefixed with the % character and work much like OS command-line calls:
they get as an argument the rest of the line, where arguments are passed without parentheses or quotes.

Cell magics are prefixed with a double %%, and they are functions that get as an argument
not only the rest of the line, but also the lines below it in a separate argument.

%run
%edit
%history

%pastebin 3 18-20 ~1/1-5
This will take line 3 and lines 18 to 20 from the current session, and lines 1-5 from the previous session.
..."""

In [1]: !ping www.bbc.co.uk
In [2]: !ls
'ls' is not recognized as an internal or external command,

In [3]: !dir
In [4]: x=4
In [5]: !echo $x
In [6]: quit

D:\ipython>

//=== http://ipython.org/ipython-doc/stable/interactive/tutorial.html
> ipython profile create
to produce the default config files.
These will be placed in ~/.ipython/profile_default


//=== http://ipython.org/ipython-doc/stable/config/intro.html#profiles

> ipython profile create foo # create the profile foo

> ipython locate profile foo

> ipython --profile=foo # start IPython using the new profile

//==  http://www.computerhope.com/prompthl.htm

myipy.bat

""" ...
chdir D:\ipython
D:
rem prompt $P$G
ipython --profile=urprofile
..."""


//=== ipython tutorial
http://ipython.org/ipython-doc/stable/interactive/tutorial.html


[Q] what's ipython ?
[Q] what's ipython notebook ?
[Q] what's ipython Qt console?



//=== ipython notebook :
0. a web app
1. turn console-based ipython to web-based
2. code + contents


http://ipython.org/ipython-doc/stable/notebook/notebook.html#introduction

* to start an ipython notebook server from the command line
> ipython notebook --profile=urprofile



*** nbconvert and nbviewer
Notebooks may be exported to a range of static formats, including HTML, Markdown, LaTeX, PDF ... , via  'nbconvert'

In effect, nbviewer is simply nbconvert as a web service ?

> ipython nbconvert notebook.ipynb
> ipython nbconvert --to html notebook.ipynb
> ipython nbconvert --to markdown notebook.ipynb --stdout



//=== how to run code in the IPython Notebook
http://ipython.org/ipython-doc/stable/notebook/notebook.html#introduction

* Shift-Enter: run cell
Execute the current cell, show output (if any), and jump to the next cell below.

* Ctrl-Enter: run cell in-place
Execute the current cell as if it were in “terminal mode”, where any output is shown, but the cursor remains in the current cell.

* Alt-Enter: run cell, insert below
Executes the current cell, shows the output, and inserts a new cell between the current cell and the cell below

* Esc and Enter: Command mode and edit mode


[Q] where does ipython notebook store the new document?
--> under the folder where you start notebook.

ssh to amazon ec2 via security group

*** using ssh to connect to amazon ec2 instance by adding inbound rules to the corresponding security group

基於安全考量，
0. 限制ssh的連入 source IP ;
1. 最好在client 自行產生key pair, 避免從server 產生再下載;
2. 避免透過網路複製 private key

//===
MyIP : will auto decide the public ip address of the client computer from which you connect to the ec2 instance

MyIP 給的是client 的public IP位址, 而非server 的ec2 elastic IP
當管理員可能從諸多不同的地點連入ec2,
MyIP提供設定security group rules的方便之門


//===
想允許ssh 使用不同的key pair 連入 ec2 ,
必須

0. 修改server端的 ~/.ssh/authorized_keys， 增加對應的public key

1. clientt端 必須擁有對應的private key ,
* git bash
$ ssh -i priv.pem ...

* putty needs priv.ppk [可用puttyGen.exe 將 pem 轉成 ppk]
Connection --> Data --> auto login username: ubuntu / ec2_user
Connection --> SSH --> Auth --> Browse for location of ppk file



//=== security group !== iptables

amazon ec2 的 security group 與 iptables 看來很像,
不過

0. security groups 是跨 instance 的 (cross-instance, instance-independent)

1. 每個 linux instance 都有各自的 iptables

2. security groups are not state-sensitive




//=== http://serverfault.com/questions/286665/why-have-both-security-groups-and-iptables-on-amazon-ec2
amazon

""" ...
I figured if anything Security Groups are just a fancy API for IPTables. It turns out they're running completely exclusively from what I can tell. Is there any reason to use both? One firewall should be plenty and adding another layer of complexity seems to be a headache just waiting to happen

...
The security groups add no load to your server - they are processed externally, and block traffic to and from your server, independent of your server. This provides an excellence first line of defense that is much more resilient than one residing on your server.

...
However, security groups are not state-sensitive, you cannot have them respond automatically to an attack for instance. IPTables are well suited to more dynamic rules - either adapting to certain scenarios, or providing finer grained conditional control.

... think about the security group like a hardware firewall in a normal networking scenario.

I guess you wouldn't really have to use both unless you had a special scenario, for example: you have a security group called webservers that controls access to web servers. You want to block an IP from hitting port 80 on one of those servers but not all of them. So what you would want to do is go into iptables on that one server

..."""

how to make notepad save file in utf-8 encoding by default

[Q] how to make the default file encoding to be utf-8 when saving a file by notepad?

0. create an empty file and save it with utf8 encoding
1. place it under Documents(or your frequent folder) and make it read-only

2. right-click to create a shortcut to it
[change the icon of the shortcut by Right-click ->Properties->ShortCut tab -> Change Icon ]

3. place the shortcut on the desktop



//=== http://answers.microsoft.com/en-us/windows/forum/windows_7-windows_programs/default-utf-8-encoding-for-new-notepad-documents/525f0ae7-121e-4eac-a6c2-cfe6b498712c

""" ...

1- Open notepad and don't type anything;

2- Go to File>Save As...;

...

4- Under "Encoding" choose "UTF-8";5- Save file in a safe place;

...

7- Right click on it and choose "Properties";

8- In "General" tab check the "Read-only" checkbox and press "OK";

9- Again right click on it and choose "Create Shortcut";

...

11- Right click on "Notepad.lnk" and choose "Properties";

12- In the "Shortcut" tab click on "Change Icon...";

13- Under "Look for icons in this file:" type the path of "notepad.exe" which is "%windir%\system32\notepad.exe" and press "OK" twice;

...

place Notepad.lnk on the desktop for more ease


... """