{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Summary of all harvested series\n",
"\n",
"**Date harvested: 10 May 2018**\n",
"\n",
"This notebook displays summary information from all 18 series in the National Archives of Australia that are listed on RecordSearch as including content recorded by the Australian Security Intelligence Organisation (ASIO).\n",
"\n",
"Item level metadata for all of these series has been harvested from RecordSearch, the National Archive's online database, and saved as CSV-formatted files.\n",
"\n",
"Just click on one of the series numbers in the table below for more detailed information, some simple visualisations, and a link to the CSV file."
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from IPython.core.display import display, HTML\n",
"import series_details\n",
"import plotly.offline as py\n",
"py.init_notebook_mode()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"series_list = ['A6119', 'A6122', 'A6126', 'A9626', 'A6335', 'B2836', 'A8703', 'A13828', 'A6281', 'A6285', 'A6283', 'A6282', 'A9106', 'A9108', 'A9105', 'A12694', 'D1902', 'D1915']"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"df = series_details.make_df_all(series_list)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/html": [
"
Aggregated totals
"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Total items | 18,777 |
---|
Access status | |
---|
Open | 4,524 (24.09%) |
OWE | 11,104 (59.14%) |
NYE | 2,891 (15.40%) |
Closed | 249 (1.33%) |
Number of items digitised | 4,300 (22.90%) |
---|
Number of pages digitised | 381,689 |
---|
Date of earliest content | 0 |
---|
Date of latest content | 2009 |
---|
"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"series_details.display_summary_all(df)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary by series"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/html": [
" \n",
" \n",
" \n",
" | \n",
" series | \n",
" total_items | \n",
" date_from | \n",
" date_to | \n",
" Open | \n",
" OWE | \n",
" NYE | \n",
" Closed | \n",
" digitised_files | \n",
" digitised_pages | \n",
" % open | \n",
" % digitised | \n",
"
\n",
" \n",
" 0 | \n",
" A6119 | \n",
" 6,741 | \n",
" 1852 | \n",
" 2009 | \n",
" 43 | \n",
" 6,314 | \n",
" 363 | \n",
" 20 | \n",
" 2,320 | \n",
" 258,547 | \n",
" 0.64% | \n",
" 34.42% | \n",
"
\n",
" 1 | \n",
" A6122 | \n",
" 2,819 | \n",
" 1800 | \n",
" 1993 | \n",
" 162 | \n",
" 2,376 | \n",
" 137 | \n",
" 138 | \n",
" 565 | \n",
" 69,007 | \n",
" 5.75% | \n",
" 20.04% | \n",
"
\n",
" 2 | \n",
" A6126 | \n",
" 1,409 | \n",
" 1800 | \n",
" 1993 | \n",
" 83 | \n",
" 1,306 | \n",
" 8 | \n",
" 11 | \n",
" 364 | \n",
" 13,521 | \n",
" 5.89% | \n",
" 25.83% | \n",
"
\n",
" 3 | \n",
" A9626 | \n",
" 1,075 | \n",
" 1919 | \n",
" 1998 | \n",
" 792 | \n",
" 277 | \n",
" 6 | \n",
" 0 | \n",
" 570 | \n",
" 9,370 | \n",
" 73.67% | \n",
" 53.02% | \n",
"
\n",
" 4 | \n",
" A6335 | \n",
" 42 | \n",
" 1922 | \n",
" 1956 | \n",
" 38 | \n",
" 4 | \n",
" 0 | \n",
" 0 | \n",
" 25 | \n",
" 2,607 | \n",
" 90.48% | \n",
" 59.52% | \n",
"
\n",
" 5 | \n",
" B2836 | \n",
" 14 | \n",
" 1926 | \n",
" 1972 | \n",
" 14 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3 | \n",
" 375 | \n",
" 100.00% | \n",
" 21.43% | \n",
"
\n",
" 6 | \n",
" A8703 | \n",
" 641 | \n",
" 1937 | \n",
" 1980 | \n",
" 328 | \n",
" 0 | \n",
" 313 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 51.17% | \n",
" 0.00% | \n",
"
\n",
" 7 | \n",
" A13828 | \n",
" 12 | \n",
" 1955 | \n",
" 1974 | \n",
" 3 | \n",
" 0 | \n",
" 9 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 25.00% | \n",
" 0.00% | \n",
"
\n",
" 8 | \n",
" A6281 | \n",
" 17 | \n",
" 0 | \n",
" 0 | \n",
" 11 | \n",
" 1 | \n",
" 5 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 64.71% | \n",
" 0.00% | \n",
"
\n",
" 9 | \n",
" A6285 | \n",
" 132 | \n",
" 1954 | \n",
" 1955 | \n",
" 83 | \n",
" 31 | \n",
" 17 | \n",
" 0 | \n",
" 110 | \n",
" 186 | \n",
" 62.88% | \n",
" 83.33% | \n",
"
\n",
" 10 | \n",
" A6283 | \n",
" 256 | \n",
" 1800 | \n",
" 1959 | \n",
" 21 | \n",
" 208 | \n",
" 24 | \n",
" 3 | \n",
" 23 | \n",
" 3,352 | \n",
" 8.20% | \n",
" 8.98% | \n",
"
\n",
" 11 | \n",
" A6282 | \n",
" 14 | \n",
" 1954 | \n",
" 1956 | \n",
" 13 | \n",
" 1 | \n",
" 0 | \n",
" 0 | \n",
" 2 | \n",
" 328 | \n",
" 92.86% | \n",
" 14.29% | \n",
"
\n",
" 12 | \n",
" A9106 | \n",
" 1 | \n",
" 1968 | \n",
" 1968 | \n",
" 1 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 100.00% | \n",
" 0.00% | \n",
"
\n",
" 13 | \n",
" A9108 | \n",
" 691 | \n",
" 1920 | \n",
" 1967 | \n",
" 220 | \n",
" 465 | \n",
" 2 | \n",
" 4 | \n",
" 107 | \n",
" 9,810 | \n",
" 31.84% | \n",
" 15.48% | \n",
"
\n",
" 14 | \n",
" A9105 | \n",
" 1 | \n",
" 1991 | \n",
" 1991 | \n",
" 1 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 100.00% | \n",
" 0.00% | \n",
"
\n",
" 15 | \n",
" A12694 | \n",
" 25 | \n",
" 1965 | \n",
" 1986 | \n",
" 5 | \n",
" 20 | \n",
" 0 | \n",
" 0 | \n",
" 8 | \n",
" 669 | \n",
" 20.00% | \n",
" 32.00% | \n",
"
\n",
" 16 | \n",
" D1902 | \n",
" 3 | \n",
" 1920 | \n",
" 1960 | \n",
" 3 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 100.00% | \n",
" 0.00% | \n",
"
\n",
" 17 | \n",
" D1915 | \n",
" 4,884 | \n",
" 1800 | \n",
" 1987 | \n",
" 2,703 | \n",
" 101 | \n",
" 2,007 | \n",
" 73 | \n",
" 203 | \n",
" 13,917 | \n",
" 55.34% | \n",
" 4.16% | \n",
"
\n",
"
"
],
"text/plain": [
""
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"series_details.display_series_all(df)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Contents dates"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"data": [
{
"name": "Digitised",
"type": "bar",
"x": [
1800,
1852,
1853,
1854,
1855,
1856,
1857,
1858,
1859,
1860,
1861,
1862,
1863,
1864,
1865,
1866,
1867,
1868,
1869,
1870,
1871,
1872,
1873,
1874,
1875,
1876,
1877,
1878,
1879,
1880,
1881,
1882,
1883,
1884,
1885,
1886,
1887,
1888,
1889,
1890,
1891,
1892,
1893,
1894,
1895,
1896,
1897,
1898,
1899,
1900,
1901,
1902,
1903,
1904,
1905,
1906,
1907,
1908,
1909,
1910,
1911,
1912,
1913,
1914,
1915,
1916,
1917,
1918,
1919,
1920,
1921,
1922,
1923,
1924,
1925,
1926,
1927,
1928,
1929,
1930,
1931,
1932,
1933,
1934,
1935,
1936,
1937,
1938,
1939,
1940,
1941,
1942,
1943,
1944,
1945,
1946,
1947,
1948,
1949,
1950,
1951,
1952,
1953,
1954,
1955,
1956,
1957,
1958,
1959,
1960,
1961,
1962,
1963,
1964,
1965,
1966,
1967,
1968,
1969,
1970,
1971,
1972,
1973,
1974,
1975,
1976,
1977,
1978,
1979,
1980,
1981,
1982,
1983,
1984,
1985,
1986,
1987,
1988,
1989,
1990,
1991,
1992,
1993,
1994,
1995,
1996,
1997,
1998
],
"y": [
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
8,
17,
28,
43,
52,
55,
64,
72,
65,
62,
67,
63,
135,
144,
155,
155,
163,
178,
198,
217,
230,
261,
316,
337,
349,
408,
485,
522,
546,
597,
606,
612,
610,
636,
740,
862,
912,
938,
997,
1042,
1199,
1090,
1054,
1054,
1081,
1066,
1051,
1082,
1060,
952,
963,
956,
963,
954,
939,
983,
949,
959,
927,
689,
587,
534,
477,
449,
438,
382,
326,
272,
230,
177,
113,
101,
62,
52,
35,
31,
11,
12,
7,
7,
6,
6,
6,
1,
1
]
},
{
"name": "Not digitised",
"type": "bar",
"x": [
1800,
1901,
1902,
1903,
1904,
1905,
1906,
1907,
1908,
1909,
1910,
1911,
1912,
1913,
1914,
1915,
1916,
1917,
1918,
1919,
1920,
1921,
1922,
1923,
1924,
1925,
1926,
1927,
1928,
1929,
1930,
1931,
1932,
1933,
1934,
1935,
1936,
1937,
1938,
1939,
1940,
1941,
1942,
1943,
1944,
1945,
1946,
1947,
1948,
1949,
1950,
1951,
1952,
1953,
1954,
1955,
1956,
1957,
1958,
1959,
1960,
1961,
1962,
1963,
1964,
1965,
1966,
1967,
1968,
1969,
1970,
1971,
1972,
1973,
1974,
1975,
1976,
1977,
1978,
1979,
1980,
1981,
1982,
1983,
1984,
1985,
1986,
1987,
1988,
1989,
1990,
1991,
1992,
1993,
1994,
1995,
1996,
1997,
1998,
1999,
2000,
2001,
2002,
2003,
2004,
2005,
2006,
2007,
2008,
2009
],
"y": [
10,
1,
1,
1,
1,
1,
1,
1,
1,
1,
3,
3,
3,
4,
23,
59,
122,
160,
216,
260,
377,
487,
349,
352,
348,
338,
431,
483,
530,
580,
657,
674,
770,
763,
772,
812,
850,
954,
928,
1137,
1464,
1621,
2120,
2995,
2756,
2275,
1932,
1930,
2105,
2435,
2456,
2504,
2582,
2657,
2919,
2770,
2442,
2522,
2642,
2531,
2229,
2164,
2052,
1773,
1631,
1525,
1350,
1331,
1312,
1333,
1379,
1238,
1134,
902,
780,
709,
647,
569,
504,
460,
396,
345,
327,
267,
196,
170,
96,
84,
64,
49,
31,
21,
9,
7,
3,
3,
3,
3,
2,
2,
2,
2,
2,
2,
2,
1,
1,
1,
1,
1
]
}
],
"layout": {
"barmode": "stack",
"title": "Content dates",
"xaxis": {
"title": "Year"
},
"yaxis": {
"title": "Number of items"
}
}
},
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"fig = series_details.plot_all_dates(series_list)\n",
"py.iplot(fig)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Access Status"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"data": [
{
"labels": [
"Open",
"OWE",
"NYE",
"Closed",
"Withheld pending agency advice"
],
"type": "pie",
"values": [
4524,
11104,
2891,
249,
9
]
}
],
"layout": {}
},
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"plot_data = series_details.plot_all_access_statuses(df)\n",
"py.iplot(plot_data)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}