\u3053\u3093\u306b\u3061\u306f\u3002Python\u3067\u30c7\u30fc\u30bf\u51e6\u7406\u3084\u5206\u6790\u3092\u884c\u3046\u4e0a\u3067\u3001\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u3092\u9ad8\u901f\u304b\u3064\u52b9\u7387\u7684\u306b\u6271\u3048\u308b\u3088\u3046\u306b\u3057\u305f\u3044\u3068\u601d\u3063\u305f\u3053\u3068\u306f\u3042\u308a\u307e\u305b\u3093\u304b\uff1f\u672c\u8a18\u4e8b\u3067\u306f\u3001\u305d\u3093\u306a\u60a9\u307f\u3092\u89e3\u6c7a\u3057\u3066\u304f\u308c\u308bPython\u30e9\u30a4\u30d6\u30e9\u30ea\u300cpyarrow\u300d\u306b\u3064\u3044\u3066\u3001\u305d\u306e\u57fa\u672c\u7684\u306a\u7279\u5fb4\u304b\u3089\u6d3b\u7528\u65b9\u6cd5\u307e\u3067\u8a73\u3057\u304f\u89e3\u8aac\u3057\u307e\u3059\u3002<\/p>\n\n\n\n
\u76ee\u6b21 <\/p>\n
pyarrow\u306f\u3001Apache Arrow\u30d7\u30ed\u30b8\u30a7\u30af\u30c8\u306e\u4e00\u74b0\u3068\u3057\u3066\u958b\u767a\u3055\u308c\u305fPython\u30e9\u30a4\u30d6\u30e9\u30ea\u3067\u3042\u308a\u3001\u9ad8\u901f\u3067\u30e1\u30e2\u30ea\u52b9\u7387\u306e\u826f\u3044\u30c7\u30fc\u30bf\u51e6\u7406\u3092\u5b9f\u73fe\u3057\u307e\u3059\u3002Apache Arrow\u306f\u3001\u7570\u306a\u308b\u30c7\u30fc\u30bf\u51e6\u7406\u30b7\u30b9\u30c6\u30e0\u9593\u3067\u30c7\u30fc\u30bf\u3092\u5171\u6709\u3059\u308b\u305f\u3081\u306e\u5171\u901a\u30d5\u30a9\u30fc\u30de\u30c3\u30c8\u3092\u63d0\u4f9b\u3057\u3001pyarrow\u306f\u3053\u306e\u30d5\u30a9\u30fc\u30de\u30c3\u30c8\u3092Python\u3067\u5229\u7528\u3067\u304d\u308b\u3088\u3046\u306b\u3057\u307e\u3059\u3002<\/p>\n\n\n\n
pyarrow\u306e\u4e3b\u306a\u7279\u5fb4\u306f\u3001NumPy\u3084pandas\u3068\u6bd4\u8f03\u3057\u3066\u975e\u5e38\u306b\u9ad8\u901f\u306a\u30c7\u30fc\u30bf\u51e6\u7406\u304c\u53ef\u80fd\u306a\u70b9\u3067\u3059\u3002\u5217\u6307\u5411\u306e\u30c7\u30fc\u30bf\u69cb\u9020\u3092\u63a1\u7528\u3059\u308b\u3053\u3068\u3067\u3001CPU\u3068\u30e1\u30e2\u30ea\u306e\u6700\u9069\u5316\u3092\u56f3\u308a\u3001\u30e1\u30e2\u30ea\u4f7f\u7528\u91cf\u3092\u6700\u5c0f\u9650\u306b\u6291\u3048\u306a\u304c\u3089\u52b9\u7387\u7684\u306a\u30c7\u30fc\u30bf\u51e6\u7406\u3092\u5b9f\u73fe\u3057\u307e\u3059\u3002\u307e\u305f\u3001Apache Parquet\u3001CSV\u3001JSON\u306a\u3069\u306e\u69d8\u3005\u306a\u30c7\u30fc\u30bf\u5f62\u5f0f\u306b\u5bfe\u5fdc\u3057\u3066\u304a\u308a\u3001\u5e45\u5e83\u3044\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u3092\u6271\u3046\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n
pyarrow\u3068pandas\u306f\u7dca\u5bc6\u306b\u9023\u643a\u3057\u3066\u304a\u308a\u3001\u76f8\u4e92\u306b\u30c7\u30fc\u30bf\u69cb\u9020\u3092\u5909\u63db\u3067\u304d\u307e\u3059\u3002\u4ee5\u4e0b\u306f\u3001pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092pyarrow\u306eTable\u306b\u5909\u63db\u3057\u3001\u518d\u3073pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u306b\u623b\u3059\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow as pa\nimport pandas as pd\n\n# pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092pyarrow\u306eTable\u306b\u5909\u63db\ndf = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})\ntable = pa.Table.from_pandas(df)\n\n# pyarrow\u306eTable\u3092pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u306b\u5909\u63db\ndf_new = table.to_pandas()<\/pre>\n\n\n\n\u3053\u306e\u3088\u3046\u306a\u9023\u643a\u306b\u3088\u308a\u3001pandas\u3067\u51e6\u7406\u3057\u3066\u3044\u305f\u30c7\u30fc\u30bf\u3092pyarrow\u3067\u9ad8\u901f\u306b\u51e6\u7406\u3057\u305f\u308a\u3001pyarrow\u3067\u8aad\u307f\u8fbc\u3093\u3060\u30c7\u30fc\u30bf\u3092pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3068\u3057\u3066\u5206\u6790\u3057\u305f\u308a\u3059\u308b\u3053\u3068\u304c\u5bb9\u6613\u306b\u306a\u308a\u307e\u3059\u3002<\/p>\n\n\n\n
pyarrow\u3092\u4f7f\u7528\u3059\u308b\u306b\u306f\u3001\u4ee5\u4e0b\u306e\u3088\u3046\u306b\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3092\u884c\u3044\u307e\u3059\u3002<\/p>\n\n\n\n
!pip install pyarrow<\/pre>\n\n\n\n\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u5f8c\u306f\u3001\u4e0a\u8a18\u306e\u30b5\u30f3\u30d7\u30eb\u30b3\u30fc\u30c9\u306e\u3088\u3046\u306bpyarrow\u3092\u30a4\u30f3\u30dd\u30fc\u30c8\u3057\u3066\u5229\u7528\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n
pyarrow\u306f\u3001\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u306e\u51e6\u7406\u306b\u9069\u3057\u3066\u304a\u308a\u3001\u30c7\u30fc\u30bf\u5206\u6790\u3084\u30de\u30b7\u30f3\u30e9\u30fc\u30cb\u30f3\u30b0\u306e\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u306b\u304a\u3051\u308b\u524d\u51e6\u7406\u3084ETL\uff08Extract, Transform, Load\uff09\u51e6\u7406\u3092\u9ad8\u901f\u5316\u3059\u308b\u305f\u3081\u306b\u6d3b\u7528\u3055\u308c\u3066\u3044\u307e\u3059\u3002\u6b21\u7bc0\u4ee5\u964d\u3067\u3001pyarrow\u306e\u5177\u4f53\u7684\u306a\u30e1\u30ea\u30c3\u30c8\u3084\u4f7f\u3044\u65b9\u306b\u3064\u3044\u3066\u8a73\u3057\u304f\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n\n\n\n
pyarrow\u3092\u4f7f\u3046\u30e1\u30ea\u30c3\u30c8\u301c\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u51e6\u7406\u306b\u304a\u3051\u308b3\u3064\u306e\u5229\u70b9\u301c<\/h2>\n\n\n\n
pyarrow\u3092\u4f7f\u7528\u3059\u308b\u6700\u5927\u306e\u30e1\u30ea\u30c3\u30c8\u306f\u3001\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u51e6\u7406\u306b\u304a\u3051\u308b\u9ad8\u901f\u6027\u3001\u30e1\u30e2\u30ea\u52b9\u7387\u3001\u67d4\u8edf\u6027\u306e3\u70b9\u3067\u3059\u3002\u4ee5\u4e0b\u306b\u3001\u305d\u308c\u305e\u308c\u306e\u5229\u70b9\u306b\u3064\u3044\u3066\u8a73\u3057\u304f\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n\n\n\n
\u9ad8\u901f\u51e6\u7406\uff1apandas\u306e\u6570\u5341\u500d\u306e\u51e6\u7406\u30b9\u30d4\u30fc\u30c9\u3092\u5b9f\u73fe<\/h3>\n\n\n\n
pyarrow\u306f\u3001pandas\u3068\u6bd4\u8f03\u3057\u3066\u6570\u5341\u500d\u304b\u3089\u6570\u767e\u500d\u306e\u51e6\u7406\u901f\u5ea6\u3092\u5b9f\u73fe\u3057\u307e\u3059\u3002\u3053\u308c\u306f\u3001Apache Arrow\u5f62\u5f0f\u306e\u30e1\u30e2\u30ea\u4e0a\u3067\u306e\u5217\u6307\u5411\u30c7\u30fc\u30bf\u69cb\u9020\u306b\u3088\u308a\u3001CPU\u3068\u30e1\u30e2\u30ea\u306e\u6700\u9069\u5316\u304c\u56f3\u3089\u308c\u3066\u3044\u308b\u305f\u3081\u3067\u3059\u3002\u307e\u305f\u3001GPU\u3084\u30de\u30eb\u30c1\u30b3\u30a2CPU\u3092\u6d3b\u7528\u3057\u305f\u4e26\u5217\u51e6\u7406\u306b\u3082\u5bfe\u5fdc\u3057\u3066\u304a\u308a\u3001\u3055\u3089\u306a\u308b\u9ad8\u901f\u5316\u304c\u53ef\u80fd\u3067\u3059\u3002<\/p>\n\n\n\n
\u4ee5\u4e0b\u306f\u3001pandas\u3068pyarrow\u306e\u51e6\u7406\u901f\u5ea6\u3092\u6bd4\u8f03\u3057\u305f\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow as pa\nimport pandas as pd\nimport numpy as np\n\n# \u5927\u898f\u6a21\u306a\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092\u751f\u6210\ndf = pd.DataFrame(np.random.randint(0, 100, size=(10000000, 3)), columns=list('abc'))\n\n# pandas\u3067\u51e6\u7406\n%time df.groupby('a').sum()\n\n# pyarrow\u3067\u51e6\u7406\ntable = pa.Table.from_pandas(df)\n%time table.group_by('a').aggregate([('b', 'sum'), ('c', 'sum')]).to_pandas()<\/pre>\n\n\n\n\u3053\u306e\u4f8b\u3067\u306f\u3001pyarrow\u3092\u4f7f\u7528\u3057\u305f\u51e6\u7406\u304cpandas\u3088\u308a\u3082\u6570\u5341\u500d\u9ad8\u901f\u3067\u3042\u308b\u3053\u3068\u304c\u308f\u304b\u308a\u307e\u3059\u3002<\/p>\n\n\n\n
\u30e1\u30e2\u30ea\u52b9\u7387\uff1a\u4f4e\u30e1\u30e2\u30ea\u74b0\u5883\u3067\u3082\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u3092\u6271\u3048\u308b<\/h3>\n\n\n\n
pyarrow\u306f\u3001\u5217\u6307\u5411\u306e\u30c7\u30fc\u30bf\u69cb\u9020\u3092\u63a1\u7528\u3057\u3066\u3044\u308b\u305f\u3081\u3001\u5fc5\u8981\u306a\u5217\u306e\u307f\u3092\u8aad\u307f\u8fbc\u3080\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u3053\u308c\u306b\u3088\u308a\u3001\u30e1\u30e2\u30ea\u4f7f\u7528\u91cf\u3092\u6700\u5c0f\u9650\u306b\u6291\u3048\u308b\u3053\u3068\u304c\u53ef\u80fd\u3067\u3059\u3002\u307e\u305f\u3001\u30c7\u30fc\u30bf\u5727\u7e2e\u6a5f\u80fd\u3082\u5099\u3048\u3066\u304a\u308a\u3001\u30e1\u30e2\u30ea\u52b9\u7387\u3092\u3055\u3089\u306b\u9ad8\u3081\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u304c\u30e1\u30e2\u30ea\u306b\u8f09\u305b\u304d\u308c\u306a\u3044\u5834\u5408\u3067\u3082\u3001\u30b9\u30c8\u30ea\u30fc\u30df\u30f3\u30b0\u51e6\u7406\u3092\u884c\u3046\u3053\u3068\u3067\u5bfe\u5fdc\u53ef\u80fd\u3067\u3059\u3002<\/p>\n\n\n\n
\u67d4\u8edf\u6027\uff1a\u591a\u69d8\u306a\u30c7\u30fc\u30bf\u5f62\u5f0f\u306e\u8aad\u307f\u66f8\u304d\u306b\u5bfe\u5fdc<\/h3>\n\n\n\n
pyarrow\u306f\u3001Apache Parquet\u3001CSV\u3001JSON\u3001Avro\u3001ORC\u306a\u3069\u3001\u69d8\u3005\u306a\u30c7\u30fc\u30bf\u5f62\u5f0f\u306b\u5bfe\u5fdc\u3057\u3066\u3044\u307e\u3059\u3002\u3053\u308c\u306b\u3088\u308a\u3001\u7570\u306a\u308b\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u304b\u3089\u306e\u30c7\u30fc\u30bf\u3092\u7d71\u4e00\u7684\u306b\u51e6\u7406\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u307e\u305f\u3001pandas\u3001NumPy\u3001PySpark\u3001TensorFlow\u306a\u3069\u3001\u4ed6\u306e\u30c7\u30fc\u30bf\u51e6\u7406\u30e9\u30a4\u30d6\u30e9\u30ea\u3068\u306e\u30b7\u30fc\u30e0\u30ec\u30b9\u306a\u9023\u643a\u304c\u53ef\u80fd\u3067\u3042\u308a\u3001\u65e2\u5b58\u306e\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u306b\u5bb9\u6613\u306b\u7d44\u307f\u8fbc\u3080\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n
\u4ee5\u4e0b\u306f\u3001pyarrow\u3092\u4f7f\u3063\u3066Apache Parquet\u30d5\u30a1\u30a4\u30eb\u3092\u8aad\u307f\u8fbc\u3080\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow.parquet as pq\n\n# Apache Parquet\u30d5\u30a1\u30a4\u30eb\u3092\u8aad\u307f\u8fbc\u3080\ntable = pq.read_table('example.parquet')\n\n# \u5fc5\u8981\u306a\u5217\u306e\u307f\u3092\u9078\u629e\nselected_columns = table.select(['column1', 'column2'])<\/pre>\n\n\n\n\u3053\u306e\u3088\u3046\u306b\u3001pyarrow\u3092\u4f7f\u3048\u3070\u3001\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u3092\u9ad8\u901f\u304b\u3064\u30e1\u30e2\u30ea\u52b9\u7387\u826f\u304f\u51e6\u7406\u3057\u306a\u304c\u3089\u3001\u69d8\u3005\u306a\u30c7\u30fc\u30bf\u5f62\u5f0f\u3084\u4ed6\u306e\u30e9\u30a4\u30d6\u30e9\u30ea\u3068\u306e\u9023\u643a\u3092\u67d4\u8edf\u306b\u884c\u3046\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u6b21\u7bc0\u3067\u306f\u3001pyarrow\u306e\u57fa\u672c\u7684\u306a\u4f7f\u3044\u65b9\u306b\u3064\u3044\u3066\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n\n\n\n
pyarrow\u306e\u57fa\u672c\u7684\u306a\u4f7f\u3044\u65b9<\/h2>\n\n\n\n
pyarrow\u3092\u4f7f\u3044\u59cb\u3081\u308b\u306b\u306f\u3001\u307e\u305a\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3068\u30bb\u30c3\u30c8\u30a2\u30c3\u30d7\u3092\u884c\u3046\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\u3002\u6b21\u306b\u3001pyarrow\u306e\u4e3b\u8981\u306a\u30c7\u30fc\u30bf\u69cb\u9020\u3092\u7406\u89e3\u3057\u3001\u305d\u308c\u3089\u3092\u4f7f\u3063\u3066\u30c7\u30fc\u30bf\u306e\u8aad\u307f\u8fbc\u307f\u3084\u66f8\u304d\u51fa\u3057\u3092\u884c\u3044\u307e\u3059\u3002\u4ee5\u4e0b\u3067\u306f\u3001\u3053\u308c\u3089\u306e\u57fa\u672c\u7684\u306a\u4f7f\u3044\u65b9\u306b\u3064\u3044\u3066\u9806\u306b\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n\n\n\n
\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3068\u30bb\u30c3\u30c8\u30a2\u30c3\u30d7<\/h3>\n\n\n\n
pyarrow\u306f\u3001pip\u307e\u305f\u306fconda\u3092\u4f7f\u3063\u3066\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u4ee5\u4e0b\u306f\u3001\u305d\u308c\u305e\u308c\u306e\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u65b9\u6cd5\u3067\u3059\u3002<\/p>\n\n\n\n
# pip\u3092\u4f7f\u3063\u305f\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\npip install pyarrow\n\n# conda\u3092\u4f7f\u3063\u305f\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\nconda install pyarrow -c conda-forge<\/pre>\n\n\n\n\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u6642\u306b\u306f\u3001\u4f9d\u5b58\u30e9\u30a4\u30d6\u30e9\u30ea\u306a\u3069\u306b\u6ce8\u610f\u304c\u5fc5\u8981\u3067\u3059\u3002\u7279\u306b\u3001Apache Arrow\u306eC++\u30e9\u30a4\u30d6\u30e9\u30ea\u304c\u5fc5\u8981\u3068\u306a\u308b\u305f\u3081\u3001\u30b7\u30b9\u30c6\u30e0\u306b\u3088\u3063\u3066\u306f\u30d3\u30eb\u30c9\u30c4\u30fc\u30eb\u30c1\u30a7\u30fc\u30f3\u304c\u5fc5\u8981\u3068\u306a\u308b\u5834\u5408\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n\n\n\n
\u4e3b\u8981\u306a\u30c7\u30fc\u30bf\u69cb\u9020\uff1aArray\u3001ChunkedArray\u3001RecordBatch\u3001Table<\/h3>\n\n\n\n
pyarrow\u306b\u306f\u3001\u4ee5\u4e0b\u306e4\u3064\u306e\u4e3b\u8981\u306a\u30c7\u30fc\u30bf\u69cb\u9020\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n\n\n\n
\n- Array\uff1a1\u6b21\u5143\u306e\u5747\u4e00\u306a\u578b\u306e\u914d\u5217<\/li>\n\n\n\n
- ChunkedArray\uff1a\u8907\u6570\u306eArray\u3092\u9023\u7d50\u3057\u305f\u3082\u306e<\/li>\n\n\n\n
- RecordBatch\uff1a\u8907\u6570\u306e\u914d\u5217\u3092\u884c\u3068\u5217\u3067\u69cb\u6210\u3057\u305f\u3082\u306e<\/li>\n\n\n\n
- Table\uff1a\u8907\u6570\u306eRecordBatch\u3092\u9023\u7d50\u3057\u305f\u3082\u306e<\/li>\n<\/ol>\n\n\n\n
\u3053\u308c\u3089\u306e\u30c7\u30fc\u30bf\u69cb\u9020\u3092\u4f7f\u3063\u3066\u3001\u30c7\u30fc\u30bf\u306e\u51e6\u7406\u3084\u5206\u6790\u3092\u884c\u3044\u307e\u3059\u3002\u4ee5\u4e0b\u306f\u3001\u305d\u308c\u305e\u308c\u306e\u30c7\u30fc\u30bf\u69cb\u9020\u306e\u4f5c\u6210\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow as pa\n\n# Array\u306e\u4f5c\u6210\narr = pa.array([1, 2, 3, 4])\n\n# ChunkedArray\u306e\u4f5c\u6210\nchunked_arr = pa.chunked_array([pa.array([1, 2]), pa.array([3, 4])])\n\n# RecordBatch\u306e\u4f5c\u6210\nbatch = pa.RecordBatch.from_arrays([pa.array([1, 2]), pa.array([\"a\", \"b\"])], names=[\"col1\", \"col2\"])\n\n# Table\u306e\u4f5c\u6210\ntable = pa.Table.from_batches([batch])<\/pre>\n\n\n\n\u30d5\u30a1\u30a4\u30eb\u306e\u8aad\u307f\u8fbc\u307f\u3068\u66f8\u304d\u51fa\u3057<\/h3>\n\n\n\n
pyarrow\u3092\u4f7f\u3063\u3066\u3001CSV\u3001JSON\u3001Apache Parquet\u3001Feather\u5f62\u5f0f\u306a\u3069\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u8aad\u307f\u8fbc\u3093\u3060\u308a\u3001\u66f8\u304d\u51fa\u3057\u305f\u308a\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u4ee5\u4e0b\u306f\u3001CSV\u30d5\u30a1\u30a4\u30eb\u306e\u8aad\u307f\u8fbc\u307f\u3068Apache Parquet\u30d5\u30a1\u30a4\u30eb\u306e\u66f8\u304d\u51fa\u3057\u306e\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow.csv as pv\nimport pyarrow.parquet as pq\n\n# CSV\u30d5\u30a1\u30a4\u30eb\u306e\u8aad\u307f\u8fbc\u307f\ncsv_table = pv.read_csv(\"example.csv\")\n\n# Apache Parquet\u30d5\u30a1\u30a4\u30eb\u306e\u66f8\u304d\u51fa\u3057\npq.write_table(csv_table, \"example.parquet\")<\/pre>\n\n\n\n\u30d5\u30a1\u30a4\u30eb\u306e\u8aad\u307f\u8fbc\u307f\u3084\u66f8\u304d\u51fa\u3057\u306e\u969b\u306b\u306f\u3001\u69d8\u3005\u306a\u30aa\u30d7\u30b7\u30e7\u30f3\u3092\u6307\u5b9a\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u4f8b\u3048\u3070\u3001CSV\u30d5\u30a1\u30a4\u30eb\u306e\u8aad\u307f\u8fbc\u307f\u6642\u306b\u306f\u3001\u533a\u5207\u308a\u6587\u5b57\u3084\u30d8\u30c3\u30c0\u30fc\u306e\u6709\u7121\u306a\u3069\u3092\u6307\u5b9a\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n
pyarrow\u3067\u306f\u3001NumPy\u306endarray\u3084pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3068\u306e\u76f8\u4e92\u5909\u63db\u3082\u7c21\u5358\u306b\u884c\u3046\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u307e\u305f\u3001\u30b9\u30ad\u30fc\u30de\u3092\u4f7f\u3063\u3066\u30c7\u30fc\u30bf\u306e\u578b\u3092\u660e\u793a\u7684\u306b\u6307\u5b9a\u3059\u308b\u3053\u3068\u3082\u53ef\u80fd\u3067\u3059\u3002<\/p>\n\n\n\n
\u4ee5\u4e0a\u304c\u3001pyarrow\u306e\u57fa\u672c\u7684\u306a\u4f7f\u3044\u65b9\u306e\u6982\u8981\u3067\u3059\u3002\u6b21\u7bc0\u3067\u306f\u3001pandas\u3068pyarrow\u3092\u9023\u643a\u3055\u305b\u308b\u65b9\u6cd5\u306b\u3064\u3044\u3066\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n\n\n\n
pandas\u3068pyarrow\u3092\u9023\u643a\u3055\u305b\u308b\u65b9\u6cd5<\/h2>\n\n\n\n
pyarrow\u3068pandas\u306f\u3001\u30b7\u30fc\u30e0\u30ec\u30b9\u306b\u9023\u643a\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u3053\u3053\u3067\u306f\u3001pyarrow\u3092\u7528\u3044\u3066pandas\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u306e\u51e6\u7406\u3092\u9ad8\u901f\u5316\u3059\u308b\u65b9\u6cd5\u3001Apache Parquet\u30d5\u30a1\u30a4\u30eb\u3092\u4ecb\u3057\u3066\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092\u4fdd\u5b58\u30fb\u8aad\u307f\u8fbc\u307f\u3059\u308b\u65b9\u6cd5\u3001\u304a\u3088\u3073\u6b20\u640d\u5024\u306e\u53d6\u308a\u6271\u3044\u306b\u3064\u3044\u3066\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n\n\n\n
pyarrow\u3092\u7528\u3044\u305fpandas\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u306e\u9ad8\u901f\u5316<\/h3>\n\n\n\n
pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092pyarrow\u306eTable\u306b\u5909\u63db\u3059\u308b\u3053\u3068\u3067\u3001\u9ad8\u901f\u306a\u51e6\u7406\u304c\u53ef\u80fd\u306b\u306a\u308a\u307e\u3059\u3002\u307e\u305f\u3001pyarrow\u306eTable\u3092pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u306b\u5909\u63db\u3059\u308b\u3053\u3068\u3067\u3001pandas\u306e\u8c4a\u5bcc\u306a\u6a5f\u80fd\u3092\u6d3b\u7528\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u3055\u3089\u306b\u3001pyarrow\u3092\u7528\u3044\u308b\u3053\u3068\u3067\u3001\u30e1\u30e2\u30ea\u4f7f\u7528\u91cf\u306e\u524a\u6e1b\u306b\u3082\u52b9\u679c\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n\n\n\n
\u4ee5\u4e0b\u306f\u3001pandas\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092pyarrow\u306eTable\u306b\u5909\u63db\u3057\u3001\u9ad8\u901f\u306b\u51e6\u7406\u3059\u308b\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow as pa\nimport pandas as pd\n\n# pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092\u4f5c\u6210\ndf = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})\n\n# pyarrow\u306eTable\u306b\u5909\u63db\ntable = pa.Table.from_pandas(df)\n\n# pyarrow\u3067\u9ad8\u901f\u306b\u51e6\u7406\nresult_table = table.group_by('a').aggregate([('b', 'sum')])\n\n# pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u306b\u623b\u3059\nresult_df = result_table.to_pandas()<\/pre>\n\n\n\nApache Parquet\u30d5\u30a1\u30a4\u30eb\u306b\u3088\u308b\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u306e\u7570\u306a\u308b\u5f62\u5f0f\u3067\u306e\u4fdd\u5b58\u3068\u8aad\u307f\u8fbc\u307f<\/h3>\n\n\n\n
pyarrow\u3092\u4f7f\u3063\u3066\u3001pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092Apache Parquet\u30d5\u30a1\u30a4\u30eb\u3068\u3057\u3066\u4fdd\u5b58\u3057\u3001\u5f8c\u3067\u8aad\u307f\u8fbc\u3080\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002Apache Parquet\u306f\u3001\u5217\u6307\u5411\u306e\u5727\u7e2e\u30d5\u30a1\u30a4\u30eb\u5f62\u5f0f\u3067\u3042\u308a\u3001\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u306e\u4fdd\u5b58\u3068\u8aad\u307f\u8fbc\u307f\u306b\u9069\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n
\u4ee5\u4e0b\u306f\u3001\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092Apache Parquet\u30d5\u30a1\u30a4\u30eb\u3068\u3057\u3066\u4fdd\u5b58\u3057\u3001\u8aad\u307f\u8fbc\u3080\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow.parquet as pq\n\n# pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092Apache Parquet\u30d5\u30a1\u30a4\u30eb\u3068\u3057\u3066\u4fdd\u5b58\ndf.to_parquet('example.parquet')\n\n# Apache Parquet\u30d5\u30a1\u30a4\u30eb\u3092pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3068\u3057\u3066\u8aad\u307f\u8fbc\u307f\ndf_loaded = pd.read_parquet('example.parquet')<\/pre>\n\n\n\n\u6b20\u640d\u5024\u306e\u53d6\u308a\u6271\u3044<\/h3>\n\n\n\n
pyarrow\u3068pandas\u9593\u3067\u30c7\u30fc\u30bf\u3092\u3084\u308a\u53d6\u308a\u3059\u308b\u969b\u306b\u306f\u3001\u6b20\u640d\u5024\u306e\u53d6\u308a\u6271\u3044\u306b\u6ce8\u610f\u304c\u5fc5\u8981\u3067\u3059\u3002pandas\u3067\u306fNaN\u3084None\u3092\u6b20\u640d\u5024\u3068\u3057\u3066\u6271\u3044\u307e\u3059\u304c\u3001pyarrow\u3067\u306fNull\u3092\u4f7f\u7528\u3057\u307e\u3059\u3002pyarrow\u306eTable\u306b\u30c7\u30fc\u30bf\u3092\u5909\u63db\u3059\u308b\u969b\u3001pandas\u306e\u6b20\u640d\u5024\u306fNull\u306b\u5909\u63db\u3055\u308c\u307e\u3059\u3002<\/p>\n\n\n\n
\u307e\u305f\u3001pyarrow\u306e\u30b9\u30ad\u30fc\u30de\u3092\u4f7f\u3063\u3066\u3001\u6b20\u640d\u5024\u3092\u660e\u793a\u7684\u306b\u6307\u5b9a\u3059\u308b\u3053\u3068\u3082\u3067\u304d\u307e\u3059\u3002\u3053\u308c\u306b\u3088\u308a\u3001\u6b20\u640d\u5024\u3092\u542b\u3080\u30c7\u30fc\u30bf\u3092\u52b9\u7387\u7684\u306b\u51e6\u7406\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n
\u4ee5\u4e0b\u306f\u3001\u6b20\u640d\u5024\u3092\u542b\u3080\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092pyarrow\u306eTable\u306b\u5909\u63db\u3057\u3001\u30b9\u30ad\u30fc\u30de\u3092\u4f7f\u3063\u3066\u6b20\u640d\u5024\u3092\u660e\u793a\u7684\u306b\u6307\u5b9a\u3059\u308b\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow as pa\nimport pandas as pd\n\n# \u6b20\u640d\u5024\u3092\u542b\u3080pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u3092\u4f5c\u6210\ndf = pd.DataFrame({'a': [1, 2, None], 'b': [4, 5, 6]})\n\n# pyarrow\u306eTable\u306b\u5909\u63db\uff08\u6b20\u640d\u5024\u306fNull\u306b\u5909\u63db\u3055\u308c\u308b\uff09\ntable = pa.Table.from_pandas(df)\n\n# \u30b9\u30ad\u30fc\u30de\u3092\u4f7f\u3063\u3066\u6b20\u640d\u5024\u3092\u660e\u793a\u7684\u306b\u6307\u5b9a\nschema = pa.schema([('a', pa.int64()), ('b', pa.int64())])\ntable_with_nulls = pa.Table.from_pandas(df, schema=schema)<\/pre>\n\n\n\npyarrow\u3068pandas\u3092\u9023\u643a\u3055\u305b\u308b\u3053\u3068\u3067\u3001\u9ad8\u901f\u304b\u3064\u52b9\u7387\u7684\u306a\u30c7\u30fc\u30bf\u51e6\u7406\u304c\u53ef\u80fd\u306b\u306a\u308a\u307e\u3059\u3002\u6b21\u7bc0\u3067\u306f\u3001pyarrow\u306e\u5b9f\u969b\u306e\u6d3b\u7528\u4e8b\u4f8b\u306b\u3064\u3044\u3066\u7d39\u4ecb\u3057\u307e\u3059\u3002<\/p>\n\n\n\n
pyarrow\u306e\u6d3b\u7528\u4e8b\u4f8b\u301c\u30d3\u30c3\u30b0\u30c7\u30fc\u30bf\u5206\u6790\u3068\u6a5f\u68b0\u5b66\u7fd2\u3078\u306e\u5fdc\u7528\u301c<\/h2>\n\n\n\n
pyarrow\u306f\u3001\u30d3\u30c3\u30b0\u30c7\u30fc\u30bf\u5206\u6790\u3084\u6a5f\u68b0\u5b66\u7fd2\u306e\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u306b\u304a\u3044\u3066\u3001\u30c7\u30fc\u30bf\u306e\u524d\u51e6\u7406\u3084\u7279\u5fb4\u91cf\u30a8\u30f3\u30b8\u30cb\u30a2\u30ea\u30f3\u30b0\u3092\u9ad8\u901f\u5316\u3059\u308b\u305f\u3081\u306b\u6d3b\u7528\u3055\u308c\u3066\u3044\u307e\u3059\u3002\u3053\u3053\u3067\u306f\u3001\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u306eETL\u51e6\u7406\u3001\u6a5f\u68b0\u5b66\u7fd2\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u306b\u304a\u3051\u308b\u30c7\u30fc\u30bf\u524d\u51e6\u7406\u3001\u30b9\u30c8\u30ea\u30fc\u30df\u30f3\u30b0\u30c7\u30fc\u30bf\u306e\u30ea\u30a2\u30eb\u30bf\u30a4\u30e0\u51e6\u7406\u306e3\u3064\u306e\u6d3b\u7528\u4e8b\u4f8b\u3092\u7d39\u4ecb\u3057\u307e\u3059\u3002<\/p>\n\n\n\n
\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u306eETL\u51e6\u7406<\/h3>\n\n\n\n
pyarrow\u3092\u4f7f\u3046\u3068\u3001CSV\u3084JSON\u30d5\u30a1\u30a4\u30eb\u306a\u3069\u306e\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u3092\u9ad8\u901f\u306b\u8aad\u307f\u8fbc\u307f\u3001\u5909\u63db\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u307e\u305f\u3001Apache Parquet\u30d5\u30a1\u30a4\u30eb\u3092\u7528\u3044\u308b\u3053\u3068\u3067\u3001\u30c7\u30fc\u30bf\u3092\u52b9\u7387\u7684\u306b\u4fdd\u5b58\u3067\u304d\u307e\u3059\u3002pandas\u306eDataFrame\u3068\u9023\u643a\u3059\u308b\u3053\u3068\u3067\u3001\u30c7\u30fc\u30bf\u306e\u30af\u30ec\u30f3\u30b8\u30f3\u30b0\u3084\u52a0\u5de5\u3082\u5bb9\u6613\u306b\u884c\u3048\u307e\u3059\u3002<\/p>\n\n\n\n
\u4ee5\u4e0b\u306f\u3001\u5927\u898f\u6a21CSV\u30d5\u30a1\u30a4\u30eb\u3092\u8aad\u307f\u8fbc\u307f\u3001Apache Parquet\u30d5\u30a1\u30a4\u30eb\u306b\u5909\u63db\u3057\u3066\u4fdd\u5b58\u3059\u308b\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow.csv as pv\nimport pyarrow.parquet as pq\n\n# \u5927\u898f\u6a21CSV\u30d5\u30a1\u30a4\u30eb\u306e\u8aad\u307f\u8fbc\u307f\ntable = pv.read_csv(\"large_dataset.csv\")\n\n# Apache Parquet\u30d5\u30a1\u30a4\u30eb\u306b\u5909\u63db\u3057\u3066\u4fdd\u5b58\npq.write_table(table, \"large_dataset.parquet\")<\/pre>\n\n\n\n\u6a5f\u68b0\u5b66\u7fd2\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u306b\u304a\u3051\u308b\u30c7\u30fc\u30bf\u524d\u51e6\u7406<\/h3>\n\n\n\n
\u6a5f\u68b0\u5b66\u7fd2\u306e\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u3067\u306f\u3001pyarrow\u3092\u4f7f\u3063\u3066\u7279\u5fb4\u91cf\u306e\u62bd\u51fa\u3084\u9078\u629e\u3092\u9ad8\u901f\u306b\u884c\u3046\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u307e\u305f\u3001\u5b66\u7fd2\u30c7\u30fc\u30bf\u3092Apache Parquet\u30d5\u30a1\u30a4\u30eb\u3068\u3057\u3066\u4fdd\u5b58\u3057\u3001\u5fc5\u8981\u306b\u5fdc\u3058\u3066\u8aad\u307f\u8fbc\u3080\u3053\u3068\u3067\u3001\u52b9\u7387\u7684\u306a\u30c7\u30fc\u30bf\u7ba1\u7406\u304c\u53ef\u80fd\u3067\u3059\u3002NumPy\u3084pandas\u3068\u9023\u643a\u3059\u308b\u3053\u3068\u3067\u3001\u6a5f\u68b0\u5b66\u7fd2\u30e2\u30c7\u30eb\u306e\u5b66\u7fd2\u3068\u8a55\u4fa1\u3092\u5186\u6ed1\u306b\u9032\u3081\u3089\u308c\u307e\u3059\u3002<\/p>\n\n\n\n
\u4ee5\u4e0b\u306f\u3001Apache Parquet\u30d5\u30a1\u30a4\u30eb\u304b\u3089\u30c7\u30fc\u30bf\u3092\u8aad\u307f\u8fbc\u307f\u3001\u7279\u5fb4\u91cf\u3068\u30bf\u30fc\u30b2\u30c3\u30c8\u3092\u5206\u5272\u3057\u3066\u3001\u5b66\u7fd2\u30c7\u30fc\u30bf\u3068\u30c6\u30b9\u30c8\u30c7\u30fc\u30bf\u306b\u5206\u5272\u3059\u308b\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow.parquet as pq\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\n# Apache Parquet\u30d5\u30a1\u30a4\u30eb\u304b\u3089\u30c7\u30fc\u30bf\u3092\u8aad\u307f\u8fbc\u3080\ntable = pq.read_table(\"processed_data.parquet\")\ndf = table.to_pandas()\n\n# \u7279\u5fb4\u91cf\u3068\u30bf\u30fc\u30b2\u30c3\u30c8\u3092\u5206\u5272\nX = df.drop(\"target\", axis=1)\ny = df[\"target\"]\n\n# \u5b66\u7fd2\u30c7\u30fc\u30bf\u3068\u30c6\u30b9\u30c8\u30c7\u30fc\u30bf\u306b\u5206\u5272\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)<\/pre>\n\n\n\n\u30b9\u30c8\u30ea\u30fc\u30df\u30f3\u30b0\u30c7\u30fc\u30bf\u306e\u30ea\u30a2\u30eb\u30bf\u30a4\u30e0\u51e6\u7406<\/h3>\n\n\n\n
pyarrow\u306f\u3001Apache Kafka\u306a\u3069\u306e\u30b9\u30c8\u30ea\u30fc\u30df\u30f3\u30b0\u30c7\u30fc\u30bf\u3092\u30ea\u30a2\u30eb\u30bf\u30a4\u30e0\u3067\u51e6\u7406\u3059\u308b\u969b\u306b\u3082\u6d3b\u7528\u3067\u304d\u307e\u3059\u3002Kafka\u304b\u3089\u8aad\u307f\u8fbc\u3093\u3060\u30c7\u30fc\u30bf\u3092pyarrow\u306e\u30c7\u30fc\u30bf\u69cb\u9020\u306b\u5909\u63db\u3059\u308b\u3053\u3068\u3067\u3001\u9ad8\u901f\u306a\u30c7\u30fc\u30bf\u96c6\u8a08\u3084\u53ef\u8996\u5316\u304c\u53ef\u80fd\u3067\u3059\u3002\u3055\u3089\u306b\u3001Apache Spark\u3068\u9023\u643a\u3059\u308b\u3053\u3068\u3067\u3001\u30b9\u30c8\u30ea\u30fc\u30df\u30f3\u30b0\u30c7\u30fc\u30bf\u51e6\u7406\u306e\u30b9\u30b1\u30fc\u30e9\u30d3\u30ea\u30c6\u30a3\u3092\u5411\u4e0a\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n
\u4ee5\u4e0b\u306f\u3001Apache Kafka\u304b\u3089\u30b9\u30c8\u30ea\u30fc\u30df\u30f3\u30b0\u30c7\u30fc\u30bf\u3092\u8aad\u307f\u8fbc\u307f\u3001\u30ea\u30a2\u30eb\u30bf\u30a4\u30e0\u3067\u51e6\u7406\u3059\u308b\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
from kafka import KafkaConsumer\nimport pyarrow as pa\n\n# Kafka\u30b3\u30f3\u30b7\u30e5\u30fc\u30de\u30fc\u306e\u8a2d\u5b9a\nconsumer = KafkaConsumer(\"topic_name\", bootstrap_servers=[\"localhost:9092\"])\n\n# \u30b9\u30c8\u30ea\u30fc\u30df\u30f3\u30b0\u30c7\u30fc\u30bf\u306e\u8aad\u307f\u8fbc\u307f\nfor msg in consumer:\n # \u30e1\u30c3\u30bb\u30fc\u30b8\u3092pyarrow\u306eRecordBatch\u306b\u5909\u63db\n batch = pa.RecordBatch.from_pandas(pd.DataFrame([msg.value]))\n\n # \u30ea\u30a2\u30eb\u30bf\u30a4\u30e0\u3067\u30c7\u30fc\u30bf\u3092\u51e6\u7406\u30fb\u96c6\u8a08\n # ...<\/pre>\n\n\n\n\u4ee5\u4e0a\u306e\u3088\u3046\u306b\u3001pyarrow\u306f\u30d3\u30c3\u30b0\u30c7\u30fc\u30bf\u5206\u6790\u3068\u6a5f\u68b0\u5b66\u7fd2\u306e\u69d8\u3005\u306a\u5834\u9762\u3067\u6d3b\u7528\u3055\u308c\u3001\u30c7\u30fc\u30bf\u51e6\u7406\u306e\u30d1\u30d5\u30a9\u30fc\u30de\u30f3\u30b9\u3092\u5927\u5e45\u306b\u5411\u4e0a\u3055\u305b\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n
\u307e\u3068\u3081\uff1a\u52b9\u7387\u7684\u306aPython\u30c7\u30fc\u30bf\u51e6\u7406\u74b0\u5883\u69cb\u7bc9\u306bpyarrow\u3092\u5f79\u7acb\u3066\u3088\u3046<\/h2>\n\n\n\n
\u672c\u8a18\u4e8b\u3067\u306f\u3001\u9ad8\u901f\u3067\u30e1\u30e2\u30ea\u52b9\u7387\u306e\u826f\u3044\u30c7\u30fc\u30bf\u51e6\u7406\u3092\u5b9f\u73fe\u3059\u308bPython\u30e9\u30a4\u30d6\u30e9\u30ea\u300cpyarrow\u300d\u306b\u3064\u3044\u3066\u8a73\u3057\u304f\u89e3\u8aac\u3057\u307e\u3057\u305f\u3002pyarrow\u306f\u3001Apache Arrow\u30d7\u30ed\u30b8\u30a7\u30af\u30c8\u306e\u4e00\u90e8\u3068\u3057\u3066\u958b\u767a\u3055\u308c\u3001columnar\u5f62\u5f0f\u306e\u30c7\u30fc\u30bf\u69cb\u9020\u3092\u63a1\u7528\u3059\u308b\u3053\u3068\u3067\u3001CPU\u3068\u30e1\u30e2\u30ea\u306e\u6700\u9069\u5316\u3092\u56f3\u3063\u3066\u3044\u307e\u3059\u3002\u307e\u305f\u3001pandas\u306a\u3069\u4ed6\u306e\u30c7\u30fc\u30bf\u51e6\u7406\u30e9\u30a4\u30d6\u30e9\u30ea\u3068\u30b7\u30fc\u30e0\u30ec\u30b9\u306b\u9023\u643a\u3067\u304d\u308b\u305f\u3081\u3001\u65e2\u5b58\u306e\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u306b\u5bb9\u6613\u306b\u7d44\u307f\u8fbc\u3080\u3053\u3068\u304c\u53ef\u80fd\u3067\u3059\u3002<\/p>\n\n\n\n
pyarrow\u3092\u6d3b\u7528\u3059\u308b\u4e0a\u3067\u91cd\u8981\u306a\u30dd\u30a4\u30f3\u30c8\u306f\u3001\u9069\u5207\u306a\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3068\u30bb\u30c3\u30c8\u30a2\u30c3\u30d7\u3001\u52b9\u679c\u7684\u306a\u30c7\u30fc\u30bf\u69cb\u9020\u306e\u9078\u629e\u3001Apache Parquet\u30d5\u30a1\u30a4\u30eb\u306e\u6d3b\u7528\u3001\u305d\u3057\u3066pandas\u3068\u306e\u9023\u643a\u306b\u3088\u308b\u76f8\u4e57\u52b9\u679c\u306e\u6700\u5927\u5316\u3067\u3059\u3002\u4ee5\u4e0b\u306f\u3001pyarrow\u3068pandas\u3092\u7d44\u307f\u5408\u308f\u305b\u305f\u52b9\u7387\u7684\u306a\u30c7\u30fc\u30bf\u51e6\u7406\u306e\u4e00\u4f8b\u3067\u3059\u3002<\/p>\n\n\n\n
import pyarrow.parquet as pq\nimport pandas as pd\n\n# Apache Parquet\u30d5\u30a1\u30a4\u30eb\u304b\u3089\u30c7\u30fc\u30bf\u3092\u8aad\u307f\u8fbc\u3080\ntable = pq.read_table(\"large_dataset.parquet\")\n\n# pandas\u306e\u30c7\u30fc\u30bf\u30d5\u30ec\u30fc\u30e0\u306b\u5909\u63db\ndf = table.to_pandas()\n\n# pandas\u3067\u52b9\u7387\u7684\u306b\u30c7\u30fc\u30bf\u51e6\u7406\nprocessed_df = df.groupby(\"category\").agg({\"value\": \"mean\"})\n\n# pyarrow\u306e\u30c6\u30fc\u30d6\u30eb\u306b\u5909\u63db\u3057\u3001Apache Parquet\u30d5\u30a1\u30a4\u30eb\u306b\u4fdd\u5b58\nprocessed_table = pa.Table.from_pandas(processed_df)\npq.write_table(processed_table, \"processed_data.parquet\")<\/pre>\n\n\n\n\u4eca\u5f8c\u3001\u30d3\u30c3\u30b0\u30c7\u30fc\u30bf\u5206\u6790\u3084\u6a5f\u68b0\u5b66\u7fd2\u306e\u5206\u91ce\u3067\u306f\u3001pyarrow\u306e\u3055\u3089\u306a\u308b\u6d3b\u7528\u304c\u671f\u5f85\u3055\u308c\u3066\u3044\u307e\u3059\u3002Python\u30c7\u30fc\u30bf\u51e6\u7406\u30a8\u30b3\u30b7\u30b9\u30c6\u30e0\u306b\u304a\u3051\u308bpyarrow\u306e\u91cd\u8981\u6027\u306f\u9ad8\u307e\u3063\u3066\u304a\u308a\u3001\u4ed6\u306e\u30c7\u30fc\u30bf\u51e6\u7406\u30c4\u30fc\u30eb\u3084\u30d5\u30ec\u30fc\u30e0\u30ef\u30fc\u30af\u3068\u306e\u9023\u643a\u3082\u5f37\u5316\u3055\u308c\u3066\u3044\u304f\u3067\u3057\u3087\u3046\u3002<\/p>\n\n\n\n
pyarrow\u3092\u5b9f\u969b\u306e\u30d7\u30ed\u30b8\u30a7\u30af\u30c8\u306b\u5c0e\u5165\u3059\u308b\u7b2c\u4e00\u6b69\u3068\u3057\u3066\u3001\u30d1\u30d5\u30a9\u30fc\u30de\u30f3\u30b9\u6539\u5584\u304c\u671f\u5f85\u3067\u304d\u308b\u51e6\u7406\u3092\u7279\u5b9a\u3057\u3001\u5b9f\u88c5\u3057\u3066\u307f\u308b\u3053\u3068\u3092\u304a\u52e7\u3081\u3057\u307e\u3059\u3002\u307e\u305f\u3001\u30b3\u30df\u30e5\u30cb\u30c6\u30a3\u3084\u30c9\u30ad\u30e5\u30e1\u30f3\u30c8\u3092\u901a\u3058\u3066\u7d99\u7d9a\u7684\u306b\u5b66\u7fd2\u3059\u308b\u3053\u3068\u3067\u3001pyarrow\u3092\u52b9\u679c\u7684\u306b\u6d3b\u7528\u3067\u304d\u308b\u3088\u3046\u306b\u306a\u308a\u307e\u3059\u3002<\/p>\n\n\n\n
\u672c\u8a18\u4e8b\u304c\u3001\u307f\u306a\u3055\u307e\u306ePython\u30c7\u30fc\u30bf\u51e6\u7406\u74b0\u5883\u306e\u69cb\u7bc9\u3068\u6700\u9069\u5316\u306b\u5f79\u7acb\u3061\u3001pyarrow\u306e\u53ef\u80fd\u6027\u3092\u6700\u5927\u9650\u306b\u5f15\u304d\u51fa\u3059\u304d\u3063\u304b\u3051\u306b\u306a\u308c\u3070\u5e78\u3044\u3067\u3059\u3002\u305c\u3072\u3001pyarrow\u3092\u6d3b\u7528\u3057\u3066\u3001\u30c7\u30fc\u30bf\u51e6\u7406\u306e\u30d1\u30d5\u30a9\u30fc\u30de\u30f3\u30b9\u3092\u98db\u8e8d\u7684\u306b\u5411\u4e0a\u3055\u305b\u3066\u304f\u3060\u3055\u3044\uff01<\/p>\n","protected":false},"excerpt":{"rendered":"
\u3053\u3093\u306b\u3061\u306f\u3002Python\u3067\u30c7\u30fc\u30bf\u51e6\u7406\u3084\u5206\u6790\u3092\u884c\u3046\u4e0a\u3067\u3001\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u3092\u9ad8\u901f\u304b\u3064\u52b9\u7387\u7684\u306b\u6271\u3048\u308b\u3088\u3046\u306b\u3057\u305f\u3044\u3068\u601d\u3063\u305f\u3053\u3068\u306f\u3042\u308a\u307e\u305b\u3093\u304b\uff1f\u672c\u8a18\u4e8b\u3067\u306f\u3001\u305d\u3093\u306a\u60a9\u307f\u3092\u89e3\u6c7a\u3057\u3066\u304f\u308c\u308bPython\u30e9\u30a4\u30d6\u30e9\u30ea\u300cpyarrow\u300d\u306b\u3064\u3044\u3066\u3001\u305d … <\/p>\n","protected":false},"author":1,"featured_media":538,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[4],"tags":[],"class_list":{"0":"post-537","1":"post","2":"type-post","3":"status-publish","4":"format-standard","5":"has-post-thumbnail","7":"category-python"},"_links":{"self":[{"href":"https:\/\/chocottopro.com\/index.php?rest_route=\/wp\/v2\/posts\/537","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/chocottopro.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/chocottopro.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/chocottopro.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/chocottopro.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=537"}],"version-history":[{"count":1,"href":"https:\/\/chocottopro.com\/index.php?rest_route=\/wp\/v2\/posts\/537\/revisions"}],"predecessor-version":[{"id":539,"href":"https:\/\/chocottopro.com\/index.php?rest_route=\/wp\/v2\/posts\/537\/revisions\/539"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/chocottopro.com\/index.php?rest_route=\/wp\/v2\/media\/538"}],"wp:attachment":[{"href":"https:\/\/chocottopro.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=537"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/chocottopro.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=537"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/chocottopro.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=537"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}