自学大数据

添加anaconda清华源

conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/

conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/

conda config --set show_channel_urls yes

conda config --show

创建虚拟环境

conda create -n your_env_name python=X.X

conda activate ml

conda install jupyter

conda install pyspark

pip install modin spark-sklearn modin[dask] -i https://pypi.tuna.tsinghua.edu.cn/simple

配置Jupyter Notebook

jupyter notebook password

Wrote hashed password to /home/hadoop/.jupyter/jupyter_notebook_config.json

生成jupyter notebook启动参数文件并修改参数

jupyter notebook --generate-config

Writing default config to: /home/hadoop/.jupyter/jupyter_notebook_config.py

vi /home/hadoop/.jupyter/jupyter_notebook_config.py

#设置访问notebook的ip，*表示所有IP

c.NotebookApp.ip='*'

#共享目录

c.NotebookApp.notebook_dir = '/home/hadoop/share'

#填写刚刚生成的密文

c.NotebookApp.password = u'argon2:$argon2id$v=19$m=10240,t=10,p=8$VXTvt5lLI5MkMWe0N4R2lw$5PPTVM9NWaYc8XtpLj3ANQ'   

#禁止notebook启动时自动打开浏览器

c.NotebookApp.open_browser = False 

#指定访问的端口，默认是8899  

c.NotebookApp.port =8899 

c.NotebookApp.allow_remote_access = True

将虚拟环境添加到jupyter notebook中

python -m ipykernel install --user --name 虚拟环境名称 --display-name “虚拟环境名称”