bacow/authority-data
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update all

Browse Source
This commit is contained in:
yiliyassh@163.com
2023-08-19 13:41:27 +08:00
parent f9b6085b3e
commit c1566c6525
7 changed files with 123 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
code/tjgb.md
View File

@@ -4,7 +4,7 @@
### 下载统计公报 :+1::+1::+1:
#### 2013~2021年
```
wget http://www.stats.gov.cn/tjsj/tjgb/ndtjgb/index.html|
wget http://www.stats.gov.cn/sj/tjgb/ndtjgb/index.html|
xargs cat index.html |
grep -E "t202|t201[4-9]"|
grep "cont_tit"|
@@ -13,28 +13,28 @@ xargs wget
```
#### 2001~2012年
```
wget http://www.stats.gov.cn/tjsj/tjgb/ndtjgb/index.html|
wget http://www.stats.gov.cn/sj/tjgb/ndtjgb/index.html|
xargs cat index.html |
grep -E "t201[0-3]|t200[0-9]"|
grep "cont_tit"|
awk -F '"' '{print "http://www.stats.gov.cn/tjsj/tjgb/ndtjgb"$2}'|
awk -F '"' '{print "http://www.stats.gov.cn/sj/tjgb/ndtjgb"$2}'|
xargs wget
```
#### 1982~2000年
```
wget http://www.stats.gov.cn/tjsj/tjgb/ndtjgb./index_1.html|
wget http://www.stats.gov.cn/sj/tjgb/ndtjgb./index_1.html|
xargs cat index_1.html |
grep "t2002"|
grep "cont_tit"|
awk -F '"' '{print "http://www.stats.gov.cn/tjsj/tjgb/ndtjgb"$2}'|
awk -F '"' '{print "http://www.stats.gov.cn/sj/tjgb/ndtjgb"$2}'|
xargs wget
```
#### 1978~1981年
```
wget http://www.stats.gov.cn/tjsj/tjgb/ndtjgb./index_2.html|
wget http://www.stats.gov.cn/sj/tjgb/ndtjgb./index_2.html|
xargs cat index_2.html|
grep "t2002"|
grep "cont_tit"|
awk -F '"' '{print "http://www.stats.gov.cn/tjsj/tjgb/ndtjgb"$2}'|
awk -F '"' '{print "http://www.stats.gov.cn/sj/tjgb/ndtjgb"$2}'|
xargs wget
```

11
code/tjgb.py
View File

@@ -30,7 +30,7 @@ def show_result(items):
curl = item[0]
cname = item[1]
# 生成完整的链接地址
#http://www.stats.gov.cn/tjsj/zxfb/201602/t20160229_1323991.html
#http://www.stats.gov.cn/sj/zxfb/201602/t20160229_1323991.html
if str(cname).find('年')>-1:
cyear = str(cname).replace('年','')
iyear = int(cyear)
@@ -38,7 +38,7 @@ def show_result(items):
curl = 'http://www.stats.gov.cn'+str(curl)
if iyear>=1978 and iyear<=2012:
curl=str(curl).replace('./','/')
curl = 'http://www.stats.gov.cn/tjsj/tjgb/ndtjgb'+str(curl)
curl = 'http://www.stats.gov.cn/sj/tjgb/ndtjgb'+str(curl)
strs = '|['+cname+']('+curl+')'
# 显示链接信息
print(strs)
@@ -64,11 +64,12 @@ def get_file(url):
# 下载（2013~2021）
# 2022-2021年
#url = 'http://www.stats.gov.cn/tjsj/tjgb/ndtjgb/index.html'
url = 'http://www.stats.gov.cn/sj/tjgb/ndtjgb/qgndtjgb/index.html'
# 1983-2001年
#url = 'http://www.stats.gov.cn/tjsj/tjgb/ndtjgb/index_1.html'
#url = 'http://www.stats.gov.cn/sj/tjgb/ndtjgb/index_1.html'
#
url = 'http://www.stats.gov.cn/tjsj/tjgb/ndtjgb/index_2.html'
#url = 'http://www.stats.gov.cn/sj/tjgb/ndtjgb/index_2.html'
# 获取信息
html = get_html(url)

79
code/tjnq.md
View File

@@ -1,85 +1,88 @@
- [**返回**](../README.md) | [**下载**](../data/data.md)
> **shell命令一句话爬虫**
### 下载中文版统计年签 :+1::+1::+1:
### 2005~2021
>wget http://www.stats.gov.cn/tjsj/ndsj/2021/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2021/"$2}'|grep -E '.jpg|.htm'|xargs wget
### 2005~2022
>wget http://www.stats.gov.cn/sj/ndsj/2022/indexch.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2022/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
>wget http://www.stats.gov.cn/sj/ndsj/2021/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2021/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2020/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2020/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2020/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2020/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2019/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2019/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2019/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2019/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2018/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2018/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2018/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2018/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2017/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2017/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2017/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2017/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2016/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2016/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2016/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2016/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2015/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2015/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2015/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2015/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2014/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2014/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2014/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2014/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2013/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2013/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2013/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2013/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2012/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2012/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2012/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2012/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2011/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2011/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2011/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2011/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2010/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2010/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2010/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2010/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2009/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2009/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2009/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2009/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2008/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2008/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2008/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2008/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2007/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2007/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2007/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2007/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2006/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2006/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2006/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2006/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2005/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2005/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2005/left.htm|xargs cat left.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2005/"$2}'|grep -E '.jpg|.htm'|xargs wget
### 2004
>wget http://www.stats.gov.cn/tjsj/ndsj/yb2004-c/left.htm|
>wget http://www.stats.gov.cn/sj/ndsj/yb2004-c/left.htm|
xargs cat left.htm|
grep href|grep html|
awk -F '"' '{print " http://www.stats.gov.cn/tjsj/ndsj/yb2004-c/"$2}'|
awk -F '"' '{print " http://www.stats.gov.cn/sj/ndsj/yb2004-c/"$2}'|
grep -E '.jpg|.htm'|
xargs wget
>
>
- [2003](http://www.stats.gov.cn/tjsj/ndsj/yearbook2003_c.pdf)
- [2003](http://www.stats.gov.cn/sj/ndsj/yearbook2003_c.pdf)
- [2002](http://www.stats.gov.cn/yearbook2001/indexC.htm)
- [2001](http://www.stats.gov.cn/tjsj/ndsj/2001c/mulu.htm)
- [2000](http://www.stats.gov.cn/tjsj/ndsj/zgnj/mulu.html)
- [2001](http://www.stats.gov.cn/sj/ndsj/2001c/mulu.htm)
- [2000](http://www.stats.gov.cn/sj/ndsj/zgnj/mulu.html)
- [1999](http://www.stats.gov.cn/yearbook/indexC.htm)
### 下载英文版统计年签 :+1::+1::+1:
### 2007~2021
>wget http://www.stats.gov.cn/tjsj/ndsj/2021/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2021/"$2}'|grep -E '.jpg|.htm'|xargs wget
>wget http://www.stats.gov.cn/sj/ndsj/2021/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2021/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2020/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2020/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2020/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2020/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2019/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2019/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2019/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2019/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2018/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2018/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2018/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2018/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2017/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2017/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2017/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2017/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2016/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2016/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2016/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2016/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2015/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2015/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2015/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2015/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2014/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2014/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2014/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2014/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2013/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2013/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2013/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2013/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2012/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2012/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2012/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2012/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2011/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2011/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2011/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2011/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2010/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2010/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2010/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2010/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2009/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2009/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2009/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2009/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2008/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2008/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2008/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2008/"$2}'|grep -E '.jpg|.htm'|xargs wget
>
> wget http://www.stats.gov.cn/tjsj/ndsj/2007/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/tjsj/ndsj/2007/"$2}'|grep -E '.jpg|.htm'|xargs wget
> wget http://www.stats.gov.cn/sj/ndsj/2007/left_.htm|xargs cat left_.htm|grep href|grep html|awk -F "'" '{print " http://www.stats.gov.cn/sj/ndsj/2007/"$2}'|grep -E '.jpg|.htm'|xargs wget
>