[{"data":1,"prerenderedAt":1078},["ShallowReactive",2],{"authors":3,"article-2022-02-10-data-processing-pipeline":331},[4,23,35,48,61,73,85,98,111,124,136,148,161,173,185,197,209,221,233,245,258,270,282,295,307,319],{"id":5,"title":6,"body":7,"description":11,"extension":14,"meta":15,"name":16,"navigation":17,"path":18,"readingTime":19,"seo":20,"stem":21,"__hash__":22},"authors\u002Fauthors\u002Falexandre-guillon.md","Software Engineer",{"type":8,"value":9,"toc":10},"minimark",[],{"title":11,"searchDepth":12,"depth":12,"links":13},"",2,[],"md",{},"Alexandre Guillon",true,"\u002Fauthors\u002Falexandre-guillon",1,{"title":6,"description":11},"authors\u002Falexandre-guillon","4tf48mjyjFNqItOHaulICbrjeCyMag1o6801uHeTz98",{"id":24,"title":6,"body":25,"description":11,"extension":14,"meta":29,"name":30,"navigation":17,"path":31,"readingTime":19,"seo":32,"stem":33,"__hash__":34},"authors\u002Fauthors\u002Falexis-ablain.md",{"type":8,"value":26,"toc":27},[],{"title":11,"searchDepth":12,"depth":12,"links":28},[],{},"Alexis Ablain","\u002Fauthors\u002Falexis-ablain",{"title":6,"description":11},"authors\u002Falexis-ablain","_SIAtB7f-39e5t3GiJof81NP47s6MGo2n4gaHkTy1uQ",{"id":36,"title":37,"body":38,"description":11,"extension":14,"meta":42,"name":43,"navigation":17,"path":44,"readingTime":19,"seo":45,"stem":46,"__hash__":47},"authors\u002Fauthors\u002Faxel-shaita.md","Engineering Manager",{"type":8,"value":39,"toc":40},[],{"title":11,"searchDepth":12,"depth":12,"links":41},[],{},"Axel Shaïta","\u002Fauthors\u002Faxel-shaita",{"title":37,"description":11},"authors\u002Faxel-shaita","fK0argUhsBkWLjpTAhY13oYLVzQthcEYkCEdtHWmIgE",{"id":49,"title":50,"body":51,"description":11,"extension":14,"meta":55,"name":56,"navigation":17,"path":57,"readingTime":19,"seo":58,"stem":59,"__hash__":60},"authors\u002Fauthors\u002Fbaptiste-faure.md","Head of Talent Acquisition",{"type":8,"value":52,"toc":53},[],{"title":11,"searchDepth":12,"depth":12,"links":54},[],{},"Baptiste Faure","\u002Fauthors\u002Fbaptiste-faure",{"title":50,"description":11},"authors\u002Fbaptiste-faure","ELisToYtcgHmgdVWZkCclTPV6exZtfyXqhpx1jjbJHs",{"id":62,"title":6,"body":63,"description":11,"extension":14,"meta":67,"name":68,"navigation":17,"path":69,"readingTime":19,"seo":70,"stem":71,"__hash__":72},"authors\u002Fauthors\u002Fbenjamin-bouillot.md",{"type":8,"value":64,"toc":65},[],{"title":11,"searchDepth":12,"depth":12,"links":66},[],{},"Benjamin Bouillot","\u002Fauthors\u002Fbenjamin-bouillot",{"title":6,"description":11},"authors\u002Fbenjamin-bouillot","tbhCFZyfTt7ZM5b5YgqQ2nhgnSTl8BweaQQryc87fHo",{"id":74,"title":37,"body":75,"description":11,"extension":14,"meta":79,"name":80,"navigation":17,"path":81,"readingTime":19,"seo":82,"stem":83,"__hash__":84},"authors\u002Fauthors\u002Fcedric-nicoloso.md",{"type":8,"value":76,"toc":77},[],{"title":11,"searchDepth":12,"depth":12,"links":78},[],{},"Cédric Nicoloso","\u002Fauthors\u002Fcedric-nicoloso",{"title":37,"description":11},"authors\u002Fcedric-nicoloso","ibSoh4VZYiWYTuLOnZTedaAfcnvet1Q9H7ogW0LgorY",{"id":86,"title":87,"body":88,"description":11,"extension":14,"meta":92,"name":93,"navigation":17,"path":94,"readingTime":19,"seo":95,"stem":96,"__hash__":97},"authors\u002Fauthors\u002Fdavid-touzet.md","Staff Engineer",{"type":8,"value":89,"toc":90},[],{"title":11,"searchDepth":12,"depth":12,"links":91},[],{},"David Touzet","\u002Fauthors\u002Fdavid-touzet",{"title":87,"description":11},"authors\u002Fdavid-touzet","dHWwnQxb1Ubt-WwXWEODGEo9AFoq1cJUhfg3kdnYSBM",{"id":99,"title":100,"body":101,"description":11,"extension":14,"meta":105,"name":106,"navigation":17,"path":107,"readingTime":19,"seo":108,"stem":109,"__hash__":110},"authors\u002Fauthors\u002Feloise-chizat.md","Data Engineer",{"type":8,"value":102,"toc":103},[],{"title":11,"searchDepth":12,"depth":12,"links":104},[],{},"Eloïse Chizat","\u002Fauthors\u002Feloise-chizat",{"title":100,"description":11},"authors\u002Feloise-chizat","Utd72Vm9qT4hh2ZbFi6a2_nXw5Wb494Ed_HL1ra5yw8",{"id":112,"title":113,"body":114,"description":11,"extension":14,"meta":118,"name":119,"navigation":17,"path":120,"readingTime":19,"seo":121,"stem":122,"__hash__":123},"authors\u002Fauthors\u002Femmanuel-auclair.md","Staff engineer",{"type":8,"value":115,"toc":116},[],{"title":11,"searchDepth":12,"depth":12,"links":117},[],{},"Emmanuel Auclair","\u002Fauthors\u002Femmanuel-auclair",{"title":113,"description":11},"authors\u002Femmanuel-auclair","MtsA8THNLEn0dTtYEIQaGwDuf7MjQL55IOeei5gugEg",{"id":125,"title":6,"body":126,"description":11,"extension":14,"meta":130,"name":131,"navigation":17,"path":132,"readingTime":19,"seo":133,"stem":134,"__hash__":135},"authors\u002Fauthors\u002Fhoreb-parraud.md",{"type":8,"value":127,"toc":128},[],{"title":11,"searchDepth":12,"depth":12,"links":129},[],{},"Horeb Parraud","\u002Fauthors\u002Fhoreb-parraud",{"title":6,"description":11},"authors\u002Fhoreb-parraud","ajjsnUX4ohZI-ghMdbb92q_taWDkKXVZSLZXoAeLQtg",{"id":137,"title":37,"body":138,"description":11,"extension":14,"meta":142,"name":143,"navigation":17,"path":144,"readingTime":19,"seo":145,"stem":146,"__hash__":147},"authors\u002Fauthors\u002Fhugo-contreras.md",{"type":8,"value":139,"toc":140},[],{"title":11,"searchDepth":12,"depth":12,"links":141},[],{},"Hugo Contreras","\u002Fauthors\u002Fhugo-contreras",{"title":37,"description":11},"authors\u002Fhugo-contreras","2nc3VMu9ASq9Z6Pwx2-7-Ye991Pww4p-UEDBQFfjF-Q",{"id":149,"title":150,"body":151,"description":11,"extension":14,"meta":155,"name":156,"navigation":17,"path":157,"readingTime":19,"seo":158,"stem":159,"__hash__":160},"authors\u002Fauthors\u002Fjulien-tassin.md","Head of Engineering",{"type":8,"value":152,"toc":153},[],{"title":11,"searchDepth":12,"depth":12,"links":154},[],{},"Julien Tassin","\u002Fauthors\u002Fjulien-tassin",{"title":150,"description":11},"authors\u002Fjulien-tassin","iUIHI7SITje38Jh9X9uvYs4-VsHx4eCdt6hAlyLFG_o",{"id":162,"title":6,"body":163,"description":11,"extension":14,"meta":167,"name":168,"navigation":17,"path":169,"readingTime":19,"seo":170,"stem":171,"__hash__":172},"authors\u002Fauthors\u002Flaurent-renard.md",{"type":8,"value":164,"toc":165},[],{"title":11,"searchDepth":12,"depth":12,"links":166},[],{},"Laurent Renard","\u002Fauthors\u002Flaurent-renard",{"title":6,"description":11},"authors\u002Flaurent-renard","5BP7Ed-pt1SQHjh0UJ1XUrlLTcdlFaDoKBCP4deHq8A",{"id":174,"title":6,"body":175,"description":11,"extension":14,"meta":179,"name":180,"navigation":17,"path":181,"readingTime":19,"seo":182,"stem":183,"__hash__":184},"authors\u002Fauthors\u002Fleo-martin.md",{"type":8,"value":176,"toc":177},[],{"title":11,"searchDepth":12,"depth":12,"links":178},[],{},"Léo Martin","\u002Fauthors\u002Fleo-martin",{"title":6,"description":11},"authors\u002Fleo-martin","eYxCHkRgbGDV7shKdTA9s7Tu0zGV4yDGFoKR5MHQntY",{"id":186,"title":6,"body":187,"description":11,"extension":14,"meta":191,"name":192,"navigation":17,"path":193,"readingTime":19,"seo":194,"stem":195,"__hash__":196},"authors\u002Fauthors\u002Floic-bousquet.md",{"type":8,"value":188,"toc":189},[],{"title":11,"searchDepth":12,"depth":12,"links":190},[],{},"Loïc Bousquet","\u002Fauthors\u002Floic-bousquet",{"title":6,"description":11},"authors\u002Floic-bousquet","ko12qZwiGL8XNjAoy9oWypPkIjr29Pbq7vhdtgldqeQ",{"id":198,"title":6,"body":199,"description":11,"extension":14,"meta":203,"name":204,"navigation":17,"path":205,"readingTime":19,"seo":206,"stem":207,"__hash__":208},"authors\u002Fauthors\u002Floic-poullain.md",{"type":8,"value":200,"toc":201},[],{"title":11,"searchDepth":12,"depth":12,"links":202},[],{},"Loïc Poullain","\u002Fauthors\u002Floic-poullain",{"title":6,"description":11},"authors\u002Floic-poullain","oRIyJhFRTqxy5dLCYQ2OnYZ1DB-gLDUM-85vTSYuTF0",{"id":210,"title":100,"body":211,"description":11,"extension":14,"meta":215,"name":216,"navigation":17,"path":217,"readingTime":19,"seo":218,"stem":219,"__hash__":220},"authors\u002Fauthors\u002Fmaud-lelu.md",{"type":8,"value":212,"toc":213},[],{"title":11,"searchDepth":12,"depth":12,"links":214},[],{},"Maud Lélu","\u002Fauthors\u002Fmaud-lelu",{"title":100,"description":11},"authors\u002Fmaud-lelu","MMbsCKuE41OMHusrl12FIEsI-Trx7l8Nn_ANhvj2_y4",{"id":222,"title":37,"body":223,"description":11,"extension":14,"meta":227,"name":228,"navigation":17,"path":229,"readingTime":19,"seo":230,"stem":231,"__hash__":232},"authors\u002Fauthors\u002Fnicolas-poirier.md",{"type":8,"value":224,"toc":225},[],{"title":11,"searchDepth":12,"depth":12,"links":226},[],{},"Nicolas Poirier","\u002Fauthors\u002Fnicolas-poirier",{"title":37,"description":11},"authors\u002Fnicolas-poirier","dXrJkYo8az4SN_D23aYc3fQ7z8s1dR2a0lt1ogjAjJs",{"id":234,"title":37,"body":235,"description":11,"extension":14,"meta":239,"name":240,"navigation":17,"path":241,"readingTime":19,"seo":242,"stem":243,"__hash__":244},"authors\u002Fauthors\u002Fraphael-sauget.md",{"type":8,"value":236,"toc":237},[],{"title":11,"searchDepth":12,"depth":12,"links":238},[],{},"Raphaël Sauget","\u002Fauthors\u002Fraphael-sauget",{"title":37,"description":11},"authors\u002Fraphael-sauget","Uri9bcq0QDuxRA0PbBoNtu7p_5L3dALu4kzcXVW0xyM",{"id":246,"title":247,"body":248,"description":11,"extension":14,"meta":252,"name":253,"navigation":17,"path":254,"readingTime":19,"seo":255,"stem":256,"__hash__":257},"authors\u002Fauthors\u002Fromain-koenig.md","Co-funder & Head of innovation",{"type":8,"value":249,"toc":250},[],{"title":11,"searchDepth":12,"depth":12,"links":251},[],{},"Romain Koenig","\u002Fauthors\u002Fromain-koenig",{"title":247,"description":11},"authors\u002Fromain-koenig","uyS8--eG2_ezyqRABcJnMJmQKKuSArhPWd14aUvFeEw",{"id":259,"title":37,"body":260,"description":11,"extension":14,"meta":264,"name":265,"navigation":17,"path":266,"readingTime":19,"seo":267,"stem":268,"__hash__":269},"authors\u002Fauthors\u002Fromaric-juniet.md",{"type":8,"value":261,"toc":262},[],{"title":11,"searchDepth":12,"depth":12,"links":263},[],{},"Romaric Juniet","\u002Fauthors\u002Fromaric-juniet",{"title":37,"description":11},"authors\u002Fromaric-juniet","4Zb2artgT-eo-PHLXi3xi4d5t7s6PfhUxeSfXIikSUY",{"id":271,"title":6,"body":272,"description":11,"extension":14,"meta":276,"name":277,"navigation":17,"path":278,"readingTime":19,"seo":279,"stem":280,"__hash__":281},"authors\u002Fauthors\u002Fstanyslas-bres.md",{"type":8,"value":273,"toc":274},[],{"title":11,"searchDepth":12,"depth":12,"links":275},[],{},"Stanyslas Bres","\u002Fauthors\u002Fstanyslas-bres",{"title":6,"description":11},"authors\u002Fstanyslas-bres","Xa0SahETuiN4q1jrmR2ych3moAqcZ2LbU7vSfEt2RuU",{"id":283,"title":284,"body":285,"description":11,"extension":14,"meta":289,"name":290,"navigation":17,"path":291,"readingTime":19,"seo":292,"stem":293,"__hash__":294},"authors\u002Fauthors\u002Ftalent-acquisition.md","Talent Acquisition",{"type":8,"value":286,"toc":287},[],{"title":11,"searchDepth":12,"depth":12,"links":288},[],{},"Équipe Talent Acquisition","\u002Fauthors\u002Ftalent-acquisition",{"description":11},"authors\u002Ftalent-acquisition","doDfE76txftQ4wIiKjJoDmSpyzSKk0tzlgVAp6-opAY",{"id":296,"title":6,"body":297,"description":11,"extension":14,"meta":301,"name":302,"navigation":17,"path":303,"readingTime":19,"seo":304,"stem":305,"__hash__":306},"authors\u002Fauthors\u002Fvictor-borg.md",{"type":8,"value":298,"toc":299},[],{"title":11,"searchDepth":12,"depth":12,"links":300},[],{},"Victor Borg","\u002Fauthors\u002Fvictor-borg",{"title":6,"description":11},"authors\u002Fvictor-borg","-Za-JweoiP6hyclue_WkxMXdRUDTczPGlJf6AZckjUc",{"id":308,"title":6,"body":309,"description":11,"extension":14,"meta":313,"name":314,"navigation":17,"path":315,"readingTime":19,"seo":316,"stem":317,"__hash__":318},"authors\u002Fauthors\u002Fvirgil-roger.md",{"type":8,"value":310,"toc":311},[],{"title":11,"searchDepth":12,"depth":12,"links":312},[],{},"Virgil Roger","\u002Fauthors\u002Fvirgil-roger",{"title":6,"description":11},"authors\u002Fvirgil-roger","DfVFe5j0bCgXeEr381ZYOM5DP4m-pWb93J9-m_muKJ0",{"id":320,"title":6,"body":321,"description":11,"extension":14,"meta":325,"name":326,"navigation":17,"path":327,"readingTime":19,"seo":328,"stem":329,"__hash__":330},"authors\u002Fauthors\u002Fyukan-zhao.md",{"type":8,"value":322,"toc":323},[],{"title":11,"searchDepth":12,"depth":12,"links":324},[],{},"Yukan Zhao","\u002Fauthors\u002Fyukan-zhao",{"title":6,"description":11},"authors\u002Fyukan-zhao","LRPHugtAJnWHsmHxy9_SR5Zas_C5p-GR_uHEs1Fhk_E",{"id":332,"title":333,"author":334,"body":335,"date":1068,"description":1069,"extension":14,"lang":1070,"meta":1071,"navigation":17,"path":1072,"published":17,"readingTime":390,"seo":1073,"stem":1074,"tags":1075,"__hash__":1077},"articles\u002Farticles\u002F2022-02-10-data-processing-pipeline.md","Data processing pipeline","laurent-renard",{"type":8,"value":336,"toc":1062},[337,341,346,349,394,397,401,404,501,504,519,527,530,533,536,591,594,598,604,631,812,815,935,938,957,960,1042,1046,1049,1058],[338,339,340],"p",{},"Building an efficient data processing pipeline that remains easy to maintain over time can be quite\nchallenging.",[342,343,345],"h2",{"id":344},"models","Models",[338,347,348],{},"Let's consider for this article the following data structure which models an accounting transaction:",[350,351,355],"pre",{"className":352,"code":353,"language":354,"meta":11,"style":11},"language-js shiki shiki-themes github-light github-dark","{\n  \"balance\": -3000,\n  \"date\": \"2021-09-01\",\n  \"category\": \"food\",\n  \"description\": \"Yummy Restaurant corp\"\n}\n","js",[356,357,358,365,370,376,382,388],"code",{"__ignoreMap":11},[359,360,362],"span",{"class":361,"line":19},"line",[359,363,364],{},"{\n",[359,366,367],{"class":361,"line":12},[359,368,369],{},"  \"balance\": -3000,\n",[359,371,373],{"class":361,"line":372},3,[359,374,375],{},"  \"date\": \"2021-09-01\",\n",[359,377,379],{"class":361,"line":378},4,[359,380,381],{},"  \"category\": \"food\",\n",[359,383,385],{"class":361,"line":384},5,[359,386,387],{},"  \"description\": \"Yummy Restaurant corp\"\n",[359,389,391],{"class":361,"line":390},6,[359,392,393],{},"}\n",[338,395,396],{},"For our system, we want to apply calculation rules such as \"Sum of all the negative balances (debit)\nfor the transactions which occurred in 2021 and labeled as 'food'\". An additional constraint is that\nthe amount of data to process can be substancial.",[342,398,400],{"id":399},"declarative-vs-imperative","Declarative vs Imperative",[338,402,403],{},"A straightforward way to implement the rule aforementioned with modern Javascript idioms would be\nsomething similar to the following snippet:",[350,405,407],{"className":352,"code":406,"language":354,"meta":11,"style":11},"const isBalanceNegative = (transaction) => transaction.balance \u003C 0;\n\nconst isIn2021 = (transaction) => new Date(transaction.date).getFullYear() === 2021;\n\nconst isCategoryFood = (transaction) => transaction.category === \"food\";\n\nconst sum = (a, b) => a + b;\n\nconst getBalanceAmount = (transaction) => Math.abs(transaction.balance);\n\nconst processingRule = (transactions) =>\n  transactions\n    .filter(isBalanceNegative)\n    .filter(isIn2021)\n    .filter(isCategoryFood)\n    .map(getBalanceAmount)\n    .reduce(sum, 0);\n",[356,408,409,414,419,424,428,433,437,443,448,454,459,465,471,477,483,489,495],{"__ignoreMap":11},[359,410,411],{"class":361,"line":19},[359,412,413],{},"const isBalanceNegative = (transaction) => transaction.balance \u003C 0;\n",[359,415,416],{"class":361,"line":12},[359,417,418],{"emptyLinePlaceholder":17},"\n",[359,420,421],{"class":361,"line":372},[359,422,423],{},"const isIn2021 = (transaction) => new Date(transaction.date).getFullYear() === 2021;\n",[359,425,426],{"class":361,"line":378},[359,427,418],{"emptyLinePlaceholder":17},[359,429,430],{"class":361,"line":384},[359,431,432],{},"const isCategoryFood = (transaction) => transaction.category === \"food\";\n",[359,434,435],{"class":361,"line":390},[359,436,418],{"emptyLinePlaceholder":17},[359,438,440],{"class":361,"line":439},7,[359,441,442],{},"const sum = (a, b) => a + b;\n",[359,444,446],{"class":361,"line":445},8,[359,447,418],{"emptyLinePlaceholder":17},[359,449,451],{"class":361,"line":450},9,[359,452,453],{},"const getBalanceAmount = (transaction) => Math.abs(transaction.balance);\n",[359,455,457],{"class":361,"line":456},10,[359,458,418],{"emptyLinePlaceholder":17},[359,460,462],{"class":361,"line":461},11,[359,463,464],{},"const processingRule = (transactions) =>\n",[359,466,468],{"class":361,"line":467},12,[359,469,470],{},"  transactions\n",[359,472,474],{"class":361,"line":473},13,[359,475,476],{},"    .filter(isBalanceNegative)\n",[359,478,480],{"class":361,"line":479},14,[359,481,482],{},"    .filter(isIn2021)\n",[359,484,486],{"class":361,"line":485},15,[359,487,488],{},"    .filter(isCategoryFood)\n",[359,490,492],{"class":361,"line":491},16,[359,493,494],{},"    .map(getBalanceAmount)\n",[359,496,498],{"class":361,"line":497},17,[359,499,500],{},"    .reduce(sum, 0);\n",[338,502,503],{},"We could have grouped the filter rules together as well",[350,505,507],{"className":352,"code":506,"language":354,"meta":11,"style":11},"const isTransactionMatching = (transaction) =>\n  [isCategoryFood, isIn2021, isBalanceNegative].every((predicate) => predicate(transaction));\n",[356,508,509,514],{"__ignoreMap":11},[359,510,511],{"class":361,"line":19},[359,512,513],{},"const isTransactionMatching = (transaction) =>\n",[359,515,516],{"class":361,"line":12},[359,517,518],{},"  [isCategoryFood, isIn2021, isBalanceNegative].every((predicate) => predicate(transaction));\n",[338,520,521,522,526],{},"You can read it almost as plain English: it is ",[523,524,525],"strong",{},"declarative"," and the different steps are\ndecomposed into various small functions. Therefore, this implementation is easy to maintain and will\nprobably stand the test of time.",[338,528,529],{},"However, this implementation may not be efficient to process a large number of transactions as it\nneeds to load all the transactions in memory and semantically recreates a new array between each\nstep of the pipeline (in practice, Javascript engines are able to optimize this kind of code).",[338,531,532],{},"Beyond the eventual problem of performance, we can note the transactions get processed in batches\nrather than one by one, which is not ideal either (imagine the last item of the stream fails at the\nfirst stage, no data gets processed at all)",[338,534,535],{},"Another approach would be to write a loop, in an imperative way, processing one item at a time:",[350,537,539],{"className":352,"code":538,"language":354,"meta":11,"style":11},"const processingRule = (transactions) => {\n  let total = 0;\n  for (const transaction of transactions) {\n    if (isCategoryFood(transaction) && isBalanceNegative(transaction) && isIn2021(transaction)) {\n      const balanceAmount = getBalanceAmount(transaction);\n      total += sum(total, balanceAmount);\n    }\n  }\n  return total;\n};\n",[356,540,541,546,551,556,561,566,571,576,581,586],{"__ignoreMap":11},[359,542,543],{"class":361,"line":19},[359,544,545],{},"const processingRule = (transactions) => {\n",[359,547,548],{"class":361,"line":12},[359,549,550],{},"  let total = 0;\n",[359,552,553],{"class":361,"line":372},[359,554,555],{},"  for (const transaction of transactions) {\n",[359,557,558],{"class":361,"line":378},[359,559,560],{},"    if (isCategoryFood(transaction) && isBalanceNegative(transaction) && isIn2021(transaction)) {\n",[359,562,563],{"class":361,"line":384},[359,564,565],{},"      const balanceAmount = getBalanceAmount(transaction);\n",[359,567,568],{"class":361,"line":390},[359,569,570],{},"      total += sum(total, balanceAmount);\n",[359,572,573],{"class":361,"line":439},[359,574,575],{},"    }\n",[359,577,578],{"class":361,"line":445},[359,579,580],{},"  }\n",[359,582,583],{"class":361,"line":450},[359,584,585],{},"  return total;\n",[359,587,588],{"class":361,"line":456},[359,589,590],{},"};\n",[338,592,593],{},"Whereas you might save some memory and processing resources, you also lose flexibility and\nreadability. Could we get the best of both worlds?",[342,595,597],{"id":596},"streams","Streams",[338,599,600,601],{},"Conceptually, we want to process the transactions one by one as they are emitted over time. We often\ncall this concept ",[523,602,603],{},"stream.",[338,605,606,607,614,615,620,621,626,627,630],{},"Whether you refer to the ",[608,609,613],"a",{"href":610,"rel":611},"https:\u002F\u002Fdeveloper.mozilla.org\u002Fen-US\u002Fdocs\u002FWeb\u002FAPI\u002FStreams_API",[612],"nofollow","DOM streams","\nor the ",[608,616,619],{"href":617,"rel":618},"https:\u002F\u002Fnodejs.org\u002Fapi\u002Fstream.html",[612],"nodejs streams",", you can abstract away the\nimplementation details by considering as (readable) stream anything that implements the\n",[608,622,625],{"href":623,"rel":624},"https:\u002F\u002Fdeveloper.mozilla.org\u002Fen-US\u002Fdocs\u002FWeb\u002FJavaScript\u002FReference\u002FGlobal_Objects\u002FSymbol\u002FasyncIterator",[612],"async iterator protocol","\nand therefore can be consumed with a basic ",[356,628,629],{},"for await"," loop. This includes simple arrays as well as\nactual nodejs readable streams for example:",[350,632,634],{"className":352,"code":633,"language":354,"meta":11,"style":11},"import { createReadStream } from \"fs\";\n\n(async () => {\n  \u002F\u002F print the chunks of a large file in nodejs\n  for await (const chunk of createReadStream(\".\u002FsomeFile.tx\")) {\n    console.log(chunk.toString(\"utf-8\"));\n  }\n\n  \u002F\u002F simple array\n  const array = [1, 2, 3, 5];\n  for await (const number of array) {\n    console.log(number);\n  }\n\n  \u002F\u002F counting down numbers emitted over time (with async generator)\n  const wait = (time = 500) =>\n    new Promise((resolve) => {\n      setTimeout(() => {\n        resolve();\n      }, time);\n    });\n\n  async function* sequence(limit = 10) {\n    while (limit >= 0) {\n      await wait();\n      yield limit;\n      limit--;\n    }\n  }\n\n  for await (const number of sequence()) {\n    console.log(number);\n  }\n})();\n",[356,635,636,641,645,650,655,660,665,669,673,678,683,688,693,697,701,706,711,716,722,728,734,740,745,751,757,763,769,775,780,785,790,796,801,806],{"__ignoreMap":11},[359,637,638],{"class":361,"line":19},[359,639,640],{},"import { createReadStream } from \"fs\";\n",[359,642,643],{"class":361,"line":12},[359,644,418],{"emptyLinePlaceholder":17},[359,646,647],{"class":361,"line":372},[359,648,649],{},"(async () => {\n",[359,651,652],{"class":361,"line":378},[359,653,654],{},"  \u002F\u002F print the chunks of a large file in nodejs\n",[359,656,657],{"class":361,"line":384},[359,658,659],{},"  for await (const chunk of createReadStream(\".\u002FsomeFile.tx\")) {\n",[359,661,662],{"class":361,"line":390},[359,663,664],{},"    console.log(chunk.toString(\"utf-8\"));\n",[359,666,667],{"class":361,"line":439},[359,668,580],{},[359,670,671],{"class":361,"line":445},[359,672,418],{"emptyLinePlaceholder":17},[359,674,675],{"class":361,"line":450},[359,676,677],{},"  \u002F\u002F simple array\n",[359,679,680],{"class":361,"line":456},[359,681,682],{},"  const array = [1, 2, 3, 5];\n",[359,684,685],{"class":361,"line":461},[359,686,687],{},"  for await (const number of array) {\n",[359,689,690],{"class":361,"line":467},[359,691,692],{},"    console.log(number);\n",[359,694,695],{"class":361,"line":473},[359,696,580],{},[359,698,699],{"class":361,"line":479},[359,700,418],{"emptyLinePlaceholder":17},[359,702,703],{"class":361,"line":485},[359,704,705],{},"  \u002F\u002F counting down numbers emitted over time (with async generator)\n",[359,707,708],{"class":361,"line":491},[359,709,710],{},"  const wait = (time = 500) =>\n",[359,712,713],{"class":361,"line":497},[359,714,715],{},"    new Promise((resolve) => {\n",[359,717,719],{"class":361,"line":718},18,[359,720,721],{},"      setTimeout(() => {\n",[359,723,725],{"class":361,"line":724},19,[359,726,727],{},"        resolve();\n",[359,729,731],{"class":361,"line":730},20,[359,732,733],{},"      }, time);\n",[359,735,737],{"class":361,"line":736},21,[359,738,739],{},"    });\n",[359,741,743],{"class":361,"line":742},22,[359,744,418],{"emptyLinePlaceholder":17},[359,746,748],{"class":361,"line":747},23,[359,749,750],{},"  async function* sequence(limit = 10) {\n",[359,752,754],{"class":361,"line":753},24,[359,755,756],{},"    while (limit >= 0) {\n",[359,758,760],{"class":361,"line":759},25,[359,761,762],{},"      await wait();\n",[359,764,766],{"class":361,"line":765},26,[359,767,768],{},"      yield limit;\n",[359,770,772],{"class":361,"line":771},27,[359,773,774],{},"      limit--;\n",[359,776,778],{"class":361,"line":777},28,[359,779,575],{},[359,781,783],{"class":361,"line":782},29,[359,784,580],{},[359,786,788],{"class":361,"line":787},30,[359,789,418],{"emptyLinePlaceholder":17},[359,791,793],{"class":361,"line":792},31,[359,794,795],{},"  for await (const number of sequence()) {\n",[359,797,799],{"class":361,"line":798},32,[359,800,692],{},[359,802,804],{"class":361,"line":803},33,[359,805,580],{},[359,807,809],{"class":361,"line":808},34,[359,810,811],{},"})();\n",[338,813,814],{},"It then becomes easy to write composable transform\u002Ffilter operators thanks to async generators:",[350,816,818],{"className":352,"code":817,"language":354,"meta":11,"style":11},"const map = (mapFn) =>\n  async function* (stream) {\n    for await (const item of stream) {\n      yield mapFn(item);\n    }\n  };\n\nconst filter = (predicate) =>\n  async function* (stream) {\n    for await (const item of stream) {\n      if (predicate(item)) {\n        yield item;\n      }\n    }\n  };\n\n\u002F\u002F fold a stream, returning a promise\nconst reduce = (reducer, init = 0) =>\n  async function (stream) {\n    let acc = init;\n    for await (const item of stream) {\n      acc = reducer(acc, item);\n    }\n    return acc;\n  };\n",[356,819,820,825,830,835,840,844,849,853,858,862,866,871,876,881,885,889,893,898,903,908,913,917,922,926,931],{"__ignoreMap":11},[359,821,822],{"class":361,"line":19},[359,823,824],{},"const map = (mapFn) =>\n",[359,826,827],{"class":361,"line":12},[359,828,829],{},"  async function* (stream) {\n",[359,831,832],{"class":361,"line":372},[359,833,834],{},"    for await (const item of stream) {\n",[359,836,837],{"class":361,"line":378},[359,838,839],{},"      yield mapFn(item);\n",[359,841,842],{"class":361,"line":384},[359,843,575],{},[359,845,846],{"class":361,"line":390},[359,847,848],{},"  };\n",[359,850,851],{"class":361,"line":439},[359,852,418],{"emptyLinePlaceholder":17},[359,854,855],{"class":361,"line":445},[359,856,857],{},"const filter = (predicate) =>\n",[359,859,860],{"class":361,"line":450},[359,861,829],{},[359,863,864],{"class":361,"line":456},[359,865,834],{},[359,867,868],{"class":361,"line":461},[359,869,870],{},"      if (predicate(item)) {\n",[359,872,873],{"class":361,"line":467},[359,874,875],{},"        yield item;\n",[359,877,878],{"class":361,"line":473},[359,879,880],{},"      }\n",[359,882,883],{"class":361,"line":479},[359,884,575],{},[359,886,887],{"class":361,"line":485},[359,888,848],{},[359,890,891],{"class":361,"line":491},[359,892,418],{"emptyLinePlaceholder":17},[359,894,895],{"class":361,"line":497},[359,896,897],{},"\u002F\u002F fold a stream, returning a promise\n",[359,899,900],{"class":361,"line":718},[359,901,902],{},"const reduce = (reducer, init = 0) =>\n",[359,904,905],{"class":361,"line":724},[359,906,907],{},"  async function (stream) {\n",[359,909,910],{"class":361,"line":730},[359,911,912],{},"    let acc = init;\n",[359,914,915],{"class":361,"line":736},[359,916,834],{},[359,918,919],{"class":361,"line":742},[359,920,921],{},"      acc = reducer(acc, item);\n",[359,923,924],{"class":361,"line":747},[359,925,575],{},[359,927,928],{"class":361,"line":753},[359,929,930],{},"    return acc;\n",[359,932,933],{"class":361,"line":759},[359,934,848],{},[338,936,937],{},"Now if we use a generic composition function, we can easily build a processing pipeline that\nabstracts away the data source as long as it matches the abstract stream interface defined above",[350,939,941],{"className":352,"code":940,"language":354,"meta":11,"style":11},"const pipe = (fns) => (arg) => fns.reduce((x, f) => f(x), arg);\n\nconst pipeline = pipe([filter(isTransactionMatching), map(getBalanceAmount), reduce(sum, 0)]);\n",[356,942,943,948,952],{"__ignoreMap":11},[359,944,945],{"class":361,"line":19},[359,946,947],{},"const pipe = (fns) => (arg) => fns.reduce((x, f) => f(x), arg);\n",[359,949,950],{"class":361,"line":12},[359,951,418],{"emptyLinePlaceholder":17},[359,953,954],{"class":361,"line":372},[359,955,956],{},"const pipeline = pipe([filter(isTransactionMatching), map(getBalanceAmount), reduce(sum, 0)]);\n",[338,958,959],{},"We managed to write our code in a declarative way while processing the transactions one by one; and\ninterestingly the pipeline does not depend on the data source:",[350,961,963],{"className":352,"code":962,"language":354,"meta":11,"style":11},"const transactions = [\n  {\n    balance: -3000,\n    date: \"2021-09-01\",\n    category: \"food\",\n    description: \"Yummy Restaurant corp\",\n  } \u002F* ... *\u002F,\n];\n\n\u002F\u002F from an array\npipeline(transactions).then(console.log).catch(console.log);\n\n\u002F\u002F from a databse cursor\npipeline(Transaction.find()).then(console.log).catch(console.log);\n\n\u002F\u002F etc\n",[356,964,965,970,975,980,985,990,995,1000,1005,1009,1014,1019,1023,1028,1033,1037],{"__ignoreMap":11},[359,966,967],{"class":361,"line":19},[359,968,969],{},"const transactions = [\n",[359,971,972],{"class":361,"line":12},[359,973,974],{},"  {\n",[359,976,977],{"class":361,"line":372},[359,978,979],{},"    balance: -3000,\n",[359,981,982],{"class":361,"line":378},[359,983,984],{},"    date: \"2021-09-01\",\n",[359,986,987],{"class":361,"line":384},[359,988,989],{},"    category: \"food\",\n",[359,991,992],{"class":361,"line":390},[359,993,994],{},"    description: \"Yummy Restaurant corp\",\n",[359,996,997],{"class":361,"line":439},[359,998,999],{},"  } \u002F* ... *\u002F,\n",[359,1001,1002],{"class":361,"line":445},[359,1003,1004],{},"];\n",[359,1006,1007],{"class":361,"line":450},[359,1008,418],{"emptyLinePlaceholder":17},[359,1010,1011],{"class":361,"line":456},[359,1012,1013],{},"\u002F\u002F from an array\n",[359,1015,1016],{"class":361,"line":461},[359,1017,1018],{},"pipeline(transactions).then(console.log).catch(console.log);\n",[359,1020,1021],{"class":361,"line":467},[359,1022,418],{"emptyLinePlaceholder":17},[359,1024,1025],{"class":361,"line":473},[359,1026,1027],{},"\u002F\u002F from a databse cursor\n",[359,1029,1030],{"class":361,"line":479},[359,1031,1032],{},"pipeline(Transaction.find()).then(console.log).catch(console.log);\n",[359,1034,1035],{"class":361,"line":485},[359,1036,418],{"emptyLinePlaceholder":17},[359,1038,1039],{"class":361,"line":491},[359,1040,1041],{},"\u002F\u002F etc\n",[342,1043,1045],{"id":1044},"conclusion","Conclusion",[338,1047,1048],{},"While the async iterators interface is a good way to model the concept of readable stream in an\nabstract way, async generators functions allow to easily define generic processing steps in a\ndeclarative way with function composition. This approach lets us gather the best of two programming\nparadigms: declarative and imperative code.",[338,1050,1051,1052,1057],{},"N.B: the content of this post is drawn from a talk of the\n",[608,1053,1056],{"href":1054,"rel":1055},"https:\u002F\u002Fwww.meetup.com\u002Fen-AU\u002FLyonJS\u002F",[612],"lyon-js meetup"," we held in our office",[1059,1060,1061],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":11,"searchDepth":12,"depth":12,"links":1063},[1064,1065,1066,1067],{"id":344,"depth":12,"text":345},{"id":399,"depth":12,"text":400},{"id":596,"depth":12,"text":597},{"id":1044,"depth":12,"text":1045},"2022-02-10","Building an efficient data processing pipeline that remains easy to maintain over time can be quite challenging.","en",{},"\u002Farticles\u002F2022-02-10-data-processing-pipeline",{"title":333,"description":1069},"articles\u002F2022-02-10-data-processing-pipeline",[1076],"Tech","g6tg4rdBJ6o5NEnSsTUJYoBS8ThRbO1BeeN-Q0l1FLY",1778159244147]