-
Notifications
You must be signed in to change notification settings - Fork 1
/
15244030785891.html
636 lines (422 loc) · 23.2 KB
/
15244030785891.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
<!doctype html>
<html class="no-js" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>
Advanced Node.js Project Structure Tutorial - Junkman
</title>
<link href="atom.xml" rel="alternate" title="Junkman" type="application/atom+xml">
<link rel="stylesheet" href="asset/css/foundation.min.css" />
<link rel="stylesheet" href="asset/css/docs.css" />
<script src="asset/js/vendor/modernizr.js"></script>
<script src="asset/js/vendor/jquery.js"></script>
<script src="asset/highlightjs/highlight.pack.js"></script>
<link href="asset/highlightjs/styles/github.css" media="screen, projection" rel="stylesheet" type="text/css">
<script>hljs.initHighlightingOnLoad();</script>
<script type="text/javascript">
function before_search(){
var searchVal = 'site:panlw.github.io ' + document.getElementById('search_input').value;
document.getElementById('search_q').value = searchVal;
return true;
}
</script>
</head>
<body class="antialiased hide-extras">
<div class="marketing off-canvas-wrap" data-offcanvas>
<div class="inner-wrap">
<nav class="top-bar docs-bar hide-for-small" data-topbar>
<section class="top-bar-section">
<div class="row">
<div style="position: relative;width:100%;"><div style="position: absolute; width:100%;">
<ul id="main-menu" class="left">
<li id=""><a target="self" href="index.html">Home</a></li>
<li id=""><a target="_self" href="archives.html">Archives</a></li>
</ul>
<ul class="right" id="search-wrap">
<li>
<form target="_blank" onsubmit="return before_search();" action="http://google.com/search" method="get">
<input type="hidden" id="search_q" name="q" value="" />
<input tabindex="1" type="search" id="search_input" placeholder="Search"/>
</form>
</li>
</ul>
</div></div>
</div>
</section>
</nav>
<nav class="tab-bar show-for-small">
<a href="javascript:void(0)" class="left-off-canvas-toggle menu-icon">
<span> Junkman</span>
</a>
</nav>
<aside class="left-off-canvas-menu">
<ul class="off-canvas-list">
<li><a href="index.html">HOME</a></li>
<li><a href="archives.html">Archives</a></li>
<li><a href="about.html">ABOUT</a></li>
<li><label>Categories</label></li>
<li><a href="Infra.html">Infra</a></li>
<li><a href="Coding.html">Coding</a></li>
<li><a href="Modeling.html">Modeling</a></li>
<li><a href="Archtecting.html">Archtecting</a></li>
</ul>
</aside>
<a class="exit-off-canvas" href="#"></a>
<section id="main-content" role="main" class="scroll-container">
<script type="text/javascript">
$(function(){
$('#menu_item_index').addClass('is_active');
});
</script>
<div class="row">
<div class="large-8 medium-8 columns">
<div class="markdown-body article-wrap">
<div class="article">
<h1>Advanced Node.js Project Structure Tutorial</h1>
<div class="read-more clearfix">
<span class="date">2018/4/22</span>
<span>posted in </span>
<span class="posted-in"><a href='Node.js.html'>Node.js</a></span>
<span class="comments">
</span>
</div>
</div><!-- article -->
<div class="article-content">
<blockquote>
<p><a href="https://blog.codeship.com/advanced-node-js-project-structure-tutorial/">https://blog.codeship.com/advanced-node-js-project-structure-tutorial/</a></p>
<p><u>This article was originally published on <a href="https://blog.risingstack.com/node-js-project-structure-tutorial-node-js-at-scale/">the RisingStack blog</a> by <a href="https://twitter.com/@tthndrs">András Tóth</a>. With their kind permission, we’re sharing it here for Codeship readers.</u></p>
</blockquote>
<p>Project structuring is an important topic because the way you bootstrap your application can determine the whole development experience throughout the life of the project.</p>
<p>In this Node.js project structure tutorial I’ll answer some of the most common questions we receive at <a href="https://trace.risingstack.com/">RisingStack</a> about structuring advanced Node applications, and help you with structuring a complex project.</p>
<p><strong>These are the goals that we are aiming for:</strong></p>
<ul>
<li> Writing an application that is easy to scale and maintain.</li>
<li> The config is well separated from the business logic.</li>
<li> Our application can consist of multiple process types.</li>
</ul>
<blockquote>
<p><strong>Node.js at Scale</strong> is <a href="http://chapters">a collection of articles</a> focusing on the needs of companies with bigger Node.js installations and advanced Node developers.</p>
</blockquote>
<h2 id="toc_0">The Node.js Project Structure</h2>
<p>Our example application is listening on Twitter tweets and tracks certain keywords. In case of a keyword match, the tweet will be sent to a RabbitMQ queue, which will be processed and saved to Redis. We will also have a REST API exposing the tweets we have saved.</p>
<p>You can take a look at the code on <a href="https://github.com/RisingStack/multi-process-nodejs-example">GitHub</a>. The file structure for this project looks like the following:</p>
<pre><code>.
|-- config
| |-- components
| | |-- common.js
| | |-- logger.js
| | |-- rabbitmq.js
| | |-- redis.js
| | |-- server.js
| | `-- twitter.js
| |-- index.js
| |-- social-preprocessor-worker.js
| |-- twitter-stream-worker.js
| `-- web.js
|-- models
| |-- redis
| | |-- index.js
| | `-- redis.js
| |-- tortoise
| | |-- index.js
| | `-- tortoise.js
| `-- twitter
| |-- index.js
| `-- twitter.js
|-- scripts
|-- test
| `-- setup.js
|-- web
| |-- middleware
| | |-- index.js
| | `-- parseQuery.js
| |-- router
| | |-- api
| | | |-- tweets
| | | | |-- get.js
| | | | |-- get.spec.js
| | | | `-- index.js
| | | `-- index.js
| | `-- index.js
| |-- index.js
| `-- server.js
|-- worker
| |-- social-preprocessor
| | |-- index.js
| | `-- worker.js
| `-- twitter-stream
| |-- index.js
| `-- worker.js
|-- index.js
`-- package.json
</code></pre>
<p>In this example we have 3 processes:</p>
<ul>
<li> <code>twitter-stream-worker</code>: The process is listening on Twitter for keywords and sends the tweets to a RabbitMQ queue.</li>
<li> <code>social-preprocessor-worker</code>: The process is listening on the RabbitMQ queue and saves the tweets to Redis and removes old ones.</li>
<li> <code>web</code>: The process is serving a REST API with a single endpoint: <code>GET /api/v1/tweets?limit&offset</code>.</li>
</ul>
<p>We will get to what differentiates a <code>web</code> and a <code>worker</code> process, but let’s start with the config.</p>
<h3 id="toc_1">How to handle different environments and configurations</h3>
<p>Load your deployment specific configurations from environment variables and never add them to the codebase as constants. These are the configurations that can vary between deployments and runtime environments, like CI, staging or production. Basically, you can have the same code running everywhere.</p>
<p>A good test for whether the config is correctly separated from the application internals is that the codebase could be made public at any moment. This means that you can be protected from accidentally leaking secrets or compromising credentials on version control.</p>
<p><a href="https://twitter.com/share?text=A+config+is+properly+separated+from+an+app%27s+internals+if+the+code+could+go+public+at+any+moment.&url=https://blog.codeship.com/advanced-node-js-project-structure-tutorial/">A config is properly separated from an app’s internals if the code could go public at any moment.</a></p>
<p><a href="https://twitter.com/share?text=A+config+is+properly+separated+from+an+app%27s+internals+if+the+code+could+go+public+at+any+moment.&url=https://blog.codeship.com/advanced-node-js-project-structure-tutorial/">Click To Tweet</a></p>
<p>The environment variables can be accessed via the <code>process.env</code> object. Keep in mind that all the values have a type of <code>String</code>, so you might need to use type conversions.</p>
<pre><code>// config/config.js
'use strict'
// required environment variables
[
'NODE_ENV',
'PORT'
].forEach((name) => {
if (!process.env[name]) {
throw new Error(`Environment variable ${name} is missing`)
}
})
const config = {
env: process.env.NODE_ENV,
logger: {
level: process.env.LOG_LEVEL || 'info',
enabled: process.env.BOOLEAN ? process.env.BOOLEAN.toLowerCase() === 'true' : false
},
server: {
port: Number(process.env.PORT)
}
// ...
}
module.exports = config
</code></pre>
<h3 id="toc_2">Config validation</h3>
<p>Validating environment variables is also a quite useful technique. It can help you catching configuration errors on startup before your application does anything else. You can read more about the benefits of early error detection of configurations by <a href="https://blog.acolyer.org/2016/11/29/early-detection-of-configuration-errors-to-reduce-failure-damage/">Adrian Colyer in this blog post</a>.</p>
<p>This is how our improved config file looks like with schema validation using the <code>joi</code> validator:</p>
<pre><code>// config/config.js
'use strict'
const joi = require('joi')
const envVarsSchema = joi.object({
NODE_ENV: joi.string()
.allow(['development', 'production', 'test', 'provision'])
.required(),
PORT: joi.number()
.required(),
LOGGER_LEVEL: joi.string()
.allow(['error', 'warn', 'info', 'verbose', 'debug', 'silly'])
.default('info'),
LOGGER_ENABLED: joi.boolean()
.truthy('TRUE')
.truthy('true')
.falsy('FALSE')
.falsy('false')
.default(true)
}).unknown()
.required()
const { error, value: envVars } = joi.validate(process.env, envVarsSchema)
if (error) {
throw new Error(`Config validation error: ${error.message}`)
}
const config = {
env: envVars.NODE_ENV,
isTest: envVars.NODE_ENV === 'test',
isDevelopment: envVars.NODE_ENV === 'development',
logger: {
level: envVars.LOGGER_LEVEL,
enabled: envVars.LOGGER_ENABLED
},
server: {
port: envVars.PORT
}
// ...
}
module.exports = config
</code></pre>
<h3 id="toc_3">Config splitting</h3>
<p>Splitting the configuration by components can be a good solution to forego a single, growing config file.</p>
<pre><code>// config/components/logger.js
'use strict'
const joi = require('joi')
const envVarsSchema = joi.object({
LOGGER_LEVEL: joi.string()
.allow(['error', 'warn', 'info', 'verbose', 'debug', 'silly'])
.default('info'),
LOGGER_ENABLED: joi.boolean()
.truthy('TRUE')
.truthy('true')
.falsy('FALSE')
.falsy('false')
.default(true)
}).unknown()
.required()
const { error, value: envVars } = joi.validate(process.env, envVarsSchema)
if (error) {
throw new Error(`Config validation error: ${error.message}`)
}
const config = {
logger: {
level: envVars.LOGGER_LEVEL,
enabled: envVars.LOGGER_ENABLED
}
}
module.exports = config
</code></pre>
<p>Then in the <code>config.js</code> file, we only need to combine the components.</p>
<pre><code>// config/config.js
'use strict'
const common = require('./components/common')
const logger = require('./components/logger')
const redis = require('./components/redis')
const server = require('./components/server')
module.exports = Object.assign({}, common, logger, redis, server)
</code></pre>
<p>You should never group your config together into “environment” specific files, like config/production.js for production. It doesn’t scale well as your app expands into more deployments over time.</p>
<p><a href="https://twitter.com/share?text=Never+group+your+config+together+into+environment+specific+files.+It+doesn%E2%80%99t+scale+well%21&url=https://blog.codeship.com/advanced-node-js-project-structure-tutorial/">Never group your config together into environment specific files. It doesn’t scale well!</a></p>
<p><a href="https://twitter.com/share?text=Never+group+your+config+together+into+environment+specific+files.+It+doesn%E2%80%99t+scale+well%21&url=https://blog.codeship.com/advanced-node-js-project-structure-tutorial/">Click To Tweet</a></p>
<h3 id="toc_4">How to organize a multi-process application</h3>
<p>The process is the main building block of a modern application. An app can have multiple stateless processes, just like in our example. HTTP requests can be handled by a web process and long-running or scheduled background tasks by a worker. They are stateless, because any data that needs to be persisted is stored in a stateful database. For this reason, adding more concurrent processes are very simple. These processes can be independently scaled based on the load or other metrics.</p>
<p>In the previous section, we saw how to break down the config into components. This comes very handy when having different process types. Each type can have its own config only requiring the components it needs, without expecting unused environment variables.</p>
<p>In the <code>config/index.j</code>s file:</p>
<pre><code>// config/index.js
'use strict'
const processType = process.env.PROCESS_TYPE
let config
try {
config = require(`./${processType}`)
} catch (ex) {
if (ex.code === 'MODULE_NOT_FOUND') {
throw new Error(`No config for process type: ${processType}`)
}
throw ex
}
module.exports = config
</code></pre>
<p>In the root <code>index.js</code> file, we start the process selected with the <code>PROCESS_TYPE</code> environment variable:</p>
<pre><code>// index.js
'use strict'
const processType = process.env.PROCESS_TYPE
if (processType === 'web') {
require('./web')
} else if (processType === 'twitter-stream-worker') {
require('./worker/twitter-stream')
} else if (processType === 'social-preprocessor-worker') {
require('./worker/social-preprocessor')
} else {
throw new Error(`${processType} is an unsupported process type. Use one of: 'web', 'twitter-stream-worker', 'social-preprocessor-worker'!`)
}
</code></pre>
<p>The nice thing about this is that we still got one application, but we have managed to split it into multiple, independent processes. Each of them can be started and scaled individually, without influencing the other parts. You can achieve this without sacrificing your DRY codebase, because parts of the code, like the models, can be shared between the different processes.</p>
<h3 id="toc_5">How to organize your test files</h3>
<p>Place your test files next to the tested modules using some kind of naming convention, like <code><module_name>.spec.js</code> and <code></module_name><module_name>.e2e.spec.js</code>. Your tests should live together with the tested modules, keeping them in sync. It would be really hard to find and maintain the tests and the corresponding functionality when the test files are completely separated from the business logic.</p>
<p><a href="https://twitter.com/share?text=Place+your+test+files+next+to+the+tested+modules+using+some+kind+of+naming+convention.&url=https://blog.codeship.com/advanced-node-js-project-structure-tutorial/">Place your test files next to the tested modules using some kind of naming convention.</a></p>
<p><a href="https://twitter.com/share?text=Place+your+test+files+next+to+the+tested+modules+using+some+kind+of+naming+convention.&url=https://blog.codeship.com/advanced-node-js-project-structure-tutorial/">Click To Tweet</a></p>
<p>A separated <code>/test</code> folder can hold all the additional test setup and utilities not used by the application itself.</p>
<h3 id="toc_6">Where to put your build and script files</h3>
<p>We tend to create a <code>/scripts</code> folder where we put our bash and node scripts for database synchronization, front-end builds and so on. This folder separates them from your application code and prevents you from putting too many script files into the root directory. List them in your <a href="https://docs.npmjs.com/misc/scripts">npm scripts</a> for easier usage.</p>
<h3 id="toc_7">Conclusion</h3>
<p>I hope you enjoyed this article on project structuring. I highly recommend to check out our previous article on the subject, where we laid out the <a href="https://blog.risingstack.com/node-hero-node-js-project-structure-tutorial/">5 fundamentals of Node.js project structuring</a>.</p>
<p>If you have any questions, please let me know in the comments. In the next chapter of the Node.js at Scale series, we’re going to dive deep into <a href="https://blog.risingstack.com/javascript-clean-coding-best-practices-node-js-at-scale/">JavaScript clean coding</a>.</p>
<p><a href="https://twitter.com/share?text=%22Advanced+Node.js+Project+Structure+Tutorial%22+via+%40tthndrs&url=https://blog.codeship.com/advanced-node-js-project-structure-tutorial/">“Advanced Node.js Project Structure Tutorial” via @tthndrs</a></p>
<p><a href="https://twitter.com/share?text=%22Advanced+Node.js+Project+Structure+Tutorial%22+via+%40tthndrs&url=https://blog.codeship.com/advanced-node-js-project-structure-tutorial/">Click To Tweet</a></summery></p>
</div>
<div class="row">
<div class="large-6 columns">
<p class="text-left" style="padding:15px 0px;">
<a href="15244093031262.html"
title="Previous Post: A GraphQL & Node.js Express Tutorial: Powerful E-Commerce with GraphCMS">« A GraphQL & Node.js Express Tutorial: Powerful E-Commerce with GraphCMS</a>
</p>
</div>
<div class="large-6 columns">
<p class="text-right" style="padding:15px 0px;">
<a href="15243906809509.html"
title="Next Post: curl 网站开发指南">curl 网站开发指南 »</a>
</p>
</div>
</div>
<div class="comments-wrap">
<div class="share-comments">
<script type="text/javascript" src="//s7.addthis.com/js/300/addthis_widget.js#pubid=ra-5ae58078c0d7b2ab"></script>
</div>
</div>
</div><!-- article-wrap -->
</div><!-- large 8 -->
<div class="large-4 medium-4 columns">
<div class="hide-for-small">
<div id="sidebar" class="sidebar">
<div id="site-info" class="site-info">
<div class="site-a-logo"><img src="./asset/img/logo.jpg" /></div>
<h1>Junkman</h1>
<div class="site-des">“拾荒者”一词来自凯文・凯利的《失控》中关于机器学习的故事（“收集癖好机”如何完成他的收集工作）。</div>
<div class="social">
<a target="_blank" class="github" target="_blank" href="https://github.com/panlw/" title="GitHub">GitHub</a>
<a target="_blank" class="rss" href="atom.xml" title="RSS">RSS</a>
</div>
</div>
<div id="site-categories" class="side-item ">
<div class="side-header">
<h2>Categories</h2>
</div>
<div class="side-content">
<p class="cat-list">
<a href="Infra.html"><strong>Infra</strong></a>
<a href="Coding.html"><strong>Coding</strong></a>
<a href="Modeling.html"><strong>Modeling</strong></a>
<a href="Archtecting.html"><strong>Archtecting</strong></a>
</p>
</div>
</div>
<div id="site-categories" class="side-item">
<div class="side-header">
<h2>Recent Posts</h2>
</div>
<div class="side-content">
<ul class="posts-list">
<li class="post">
<a href="15517999043443.html">The Art of Crafting Architectural Diagrams</a>
</li>
<li class="post">
<a href="15517997955971.html">为什么说我们需要软件架构图？</a>
</li>
<li class="post">
<a href="15516128677869.html">DNS Servers That Offer Privacy and Filtering</a>
</li>
<li class="post">
<a href="15516123108194.html">Airbnb's Migration from Monolith to Services</a>
</li>
<li class="post">
<a href="15516097487470.html">Events As First-Class Citizens</a>
</li>
</ul>
</div>
</div>
</div><!-- sidebar -->
</div><!-- hide for small -->
</div><!-- large 4 -->
</div><!-- row -->
<div class="page-bottom clearfix">
<div class="row">
<p class="copyright">Copyright © 2015
Powered by <a target="_blank" href="http://www.mweb.im">MWeb</a>,
Theme used <a target="_blank" href="http://github.com">GitHub CSS</a>.</p>
</div>
</div>
</section>
</div>
</div>
<script src="asset/js/foundation.min.js"></script>
<script>
$(document).foundation();
function fixSidebarHeight(){
var w1 = $('.markdown-body').height();
var w2 = $('#sidebar').height();
if (w1 > w2) { $('#sidebar').height(w1); };
}
$(function(){
fixSidebarHeight();
})
$(window).load(function(){
fixSidebarHeight();
});
</script>
<script src="asset/chart/all-min.js"></script><script type="text/javascript">$(function(){ var mwebii=0; var mwebChartEleId = 'mweb-chart-ele-'; $('pre>code').each(function(){ mwebii++; var eleiid = mwebChartEleId+mwebii; if($(this).hasClass('language-sequence')){ var ele = $(this).addClass('nohighlight').parent(); $('<div id="'+eleiid+'"></div>').insertAfter(ele); ele.hide(); var diagram = Diagram.parse($(this).text()); diagram.drawSVG(eleiid,{theme: 'simple'}); }else if($(this).hasClass('language-flow')){ var ele = $(this).addClass('nohighlight').parent(); $('<div id="'+eleiid+'"></div>').insertAfter(ele); ele.hide(); var diagram = flowchart.parse($(this).text()); diagram.drawSVG(eleiid); } });});</script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script type="text/x-mathjax-config">MathJax.Hub.Config({TeX: { equationNumbers: { autoNumber: "AMS" } }});</script>
</body>
</html>