1<!DOCTYPE > 2<!-- paulirish.com/2008/conditional-stylesheets-vs-css-hacks-answer-neither/ --> 3<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]--> 4<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]--> 5<!--[if IE 8]> <html class="no-js lt-ie9" lang="en"> <![endif]--> 6<!-- Consider adding a manifest.appcache: h5bp.com/d/Offline --> 7<!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]--> 8<head> 9<meta charset="utf-8"></meta> 10<!-- Use the .htaccess and remove these lines to avoid edge case issues. More info: h5bp.com/i/378 --> 11<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"></meta> 12<meta name="viewport" content="width=device-width, initial-scale=1.0"></meta> 13 14 15 16 17 18<title>Self-hosted metasearch engine protects against tracking from Google, Bing, etc</title> 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41<meta name="pagename" content="Self-hosted metasearch engine protects against tracking from Google, Bing, etc"></meta> 42 43 44 45 46 47 48 49 50 51 52 53<meta name="DC.title" content="Self-hosted metasearch engine protects against tracking from Google, Bing, etc"></meta> 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68<!-- OpenGraph Meta Tags --> 69 70<meta name="og:title" content="Self-hosted metasearch engine protects against tracking from Google, Bing, etc"></meta> 71 72<meta name="og:url" content="https://techsparx.com/software-development/docker/tutorials/diy-search-engine.html"></meta> 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107<link rel="canonical" href="https://techsparx.com/software-development/docker/tutorials/diy-search-engine.html"></link> 108 109 110<link rel="sitemap" type="application/xml" href="../../../sitemap.xml" title="Sitemap"></link> 111 112 113 114 115 116 117 118 119 120<link rel="stylesheet" type="text/css" href="../../../vendor/@akashacms/diagrams-maker/style.css"></link><link rel="stylesheet" type="text/css" href="../../../akasha/epub-website/style.css"></link><link rel="stylesheet" type="text/css" href="../../../vendor/bootstrap/css/bootstrap.min.css"></link><link rel="stylesheet" type="text/css" href="../../../vendor/bootswatch-litera.css"></link><link rel="stylesheet" type="text/css" href="../../../style.css"></link><link rel="stylesheet" type="text/css" href="../../../vendor/highlight.js/styles/tomorrow-night-blue.css"></link><link rel="stylesheet" type="text/css" href="../../../vendor/fontawesome-free/css/all.min.css"></link> 121 122 123 124<link rel="alternate" type="application/rss+xml" href="../../../ALL.xml"></link> 125<link rel="alternate" type="application/rss+xml" href="../../../blog/rss.xml"></link> 126<link rel="alternate" type="application/rss+xml" href="../../../doctor-who/rss.xml"></link> 127<link rel="alternate" type="application/rss+xml" href="../../../nodejs/rss.xml"></link> 128 129<!-- Propeller Ads --> 130<meta name="propeller" content="0e6c4d6a2c29fc6abe531d0387b45dd5"></meta> 131<meta name="og:image" content="https://techsparx.com/software-development/docker/tutorials/img/open-source-search.png"></meta><meta name="og:image" content="https://techsparx.com/software-development/docker/tutorials/img/searx-alternative.png"></meta><meta name="og:image" content="https://techsparx.com/software-development/docker/tutorials/img/what-is-reiki.png"></meta><meta name="og:image" content="https://techsparx.com/software-development/docker/tutorials/img/metasearch-wtf.png"></meta><meta name="og:image" content="https://techsparx.com/software-development/docker/tutorials/img/searx2.png"></meta><meta name="og:image" content="https://techsparx.com/software-development/docker/tutorials/img/add-search-engine.png"></meta><meta name="og:image" content="https://techsparx.com/software-development/docker/tutorials/img/searxng-timeout.png"></meta><meta name="og:image" content="https://techsparx.com/img/headshot-new.jpg"></meta><meta name="og:image" content="https://techsparx.com/imgz/udemy/39197_1387.png"></meta><meta name="og:image" content="https://techsparx.com/imgz/udemy/39197_7119.png"></meta><meta name="og:image" content="https://techsparx.com/imgz/a2hosting/468x60.jpg"></meta><meta name="og:image" content="https://techsparx.com/imgz/a2hosting/1d7c1fc2.png"></meta></head> 132<body> 133 134<div class="container-fluid" role="main"> 135<nav class="navbar navbar-expand-lg navbar-light bg-light fixed-top" role="navigation"> 136 <a class="navbar-brand" href="../../../index.html"> 137 <img style="height: 50px;" id="site-logo-image" src="../../../logo.jpg" title="TechSparx News"></img> 138 </a> 139 140 <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#techsparx-navbar-collapse-1" aria-controls="techsparx-navbar-collapse-1" aria-expanded="false" aria-label="Toggle navigation">
141 <span class="navbar-toggler-icon"></span> 142 </button> 143 144 <div class="collapse navbar-collapse" id="techsparx-navbar-collapse-1"> 145 <ul class="navbar-nav mr-auto"> 146 <li class="nav-item dropdown"> 147 <a href="#" class="nav-link dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Blog <span class="caret"></span></a> 148 <div class="dropdown-menu"> 149 <a class="dropdown-item" href="../../../blog/index.html">News</a> 150 <a class="dropdown-item" href="../../../doctor-who/index.html">Doctor Who</a> 151 </div> 152 </li> 153 <li class="nav-item dropdown"> 154 <a href="#" class="nav-link dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Computers <span class="caret"></span></a> 155 <div class="dropdown-menu"> 156 <a class="dropdown-item" href="../../../homelab/index.html">Homelab self-hosted services</a> 157 <a class="dropdown-item" href="../../../chromebooks/index.html">Chromebooks</a> 158 <a class="dropdown-item" href="../../../computer-hardware/index.html">Hardware</a> 159 <a class="dropdown-item" href="../../../computer-hardware/apple/index.html">Apple</a> 160 <a class="dropdown-item" href="../../../computer-hardware/arduino/index.html">Arduino</a> 161 <a class="dropdown-item" href="../../../linux-sbc/index.html">Linux SBC's</a> 162 <a class="dropdown-item" href="../../../linux/index.html">Linux</a> 163 </div> 164 </li> 165 <li class="nav-item dropdown"> 166 <a href="#" class="nav-link dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Privacy <span class="caret"></span></a> 167 <div class="dropdown-menu"> 168 <a class="dropdown-item" href="../../../social-media-warfare/index.html">Social Media Warfare</a> 169 <a class="dropdown-item" href="../../../spying/index.html">Spying</a> 170 </div> 171 </li> 172 <li class="nav-item dropdown"> 173 <a href="#" class="nav-link dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Software Development <span class="caret"></span></a> 174 <div class="dropdown-menu"> 175 <a class="dropdown-item" href="../../spring/index.html">Spring</a> 176 <a class="dropdown-item" href="../index.html">Docker</a> 177 <a class="dropdown-item" href="../../aws/index.html">AWS</a> 178 <a class="dropdown-item" href="../../serverless-computing/index.html">Serverless</a> 179 <a class="dropdown-item" href="../../../nodejs/index.html">Node.js</a> 180 </div> 181 </li> 182 <li class="nav-item dropdown"> 183 <a href="#" class="nav-link dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Catalog <span class="caret"></span></a> 184 <div class="dropdown-menu"> 185 <a class="dropdown-item" href="../../../nodejs/learning/products.html">Learn Node.js Programming</a> 186 <a class="dropdown-item" href="../learning.html">Learn Docker</a> 187 <a class="dropdown-item" href="../../../computer-hardware/arduino/products.html">Arduino</a> 188 <a class="dropdown-item" href="../../../linux-sbc/raspberry-pi/products.html">Raspberry Pi</a> 189 </div> 190 </li> 191 <li> 192 <a href="../../../about.html" class="nav-link">About</a> 193 </li> 194 </ul> 195 <form method="get" id="cse-search-box" class="form-inline my-2 my-lg-0" role="search" action="https://duckduckgo.com/"> 196 <input type="hidden" name="sites" value="techsparx.com"></input> 197 <input type="hidden" name="k7" value="#ffffff"></input> 198 <input type="hidden" name="k8" value="#222222"></input> 199 <input type="hidden" name="k9" value="#00278e"></input> 200 <input type="hidden" name="kx" value="#20692b"></input> 201 <input type="hidden" name="kj" value="#fafafa"></input> 202 <input type="hidden" name="kt" value="p"></input> 203 <div class="form-group"> 204 <input type="text" name="q" placeholder="Search TechSparx.com" aria-label="Search techsparx.com on DuckDuckGo"></input> 205 </div> 206 <button class="btn btn-outline-success mb-2" type="submit">Search</button> 207 </form> 208 209 <!-- <form action="//www.google.com" id="cse-search-box" class="form-inline my-2 my-lg-0" role="search"> 210 <input type="hidden" name="cx" value="partner-pub-2401070691062219:9544629023" /> 211 <input type="hidden" name="ie" value="UTF-8" /> 212 <div class="form-group"> 213 <input type="text" class="form-control" placeholder="Search" name="q" /> 214 </div> 215 <button type="submit" name="sa" value="Search" class="btn btn-outline-success mb-2">Submit</button> 216
216<script type="text/javascript" src="//www.google.com/coop/cse/brand?form=cse-search-box&lang=en"></script>
216 217 </form> --> 218 219 </div> 220 221</nav> 222 223 224<div class="row"> 225 <!-- include our document's content --> 226 <article class="col-md-9"> 227 228 <div class="article-heading"> 229 <h1 class="article-title" align="center">Self-hosted metasearch engine protects against tracking from Google, Bing, etc 230 </h1></div> 231 232 <div class="article-body"> 233 234 <div id="breadcrumbTrail"> 235 236 » <a class="p-category" href="../../../index.html">TechSparx</a> 237 238 » <a class="p-category" href="../../index.html">Software Development</a> 239 240 » <a class="p-category" href="../index.html">Using Docker to deploy applications, encapsulate software tools, and otherwise simplify software development</a> 241 242 » <a class="p-category" href="index.html">Tutorials</a> 243 244 » <a class="p-category" href="diy-search-engine.html">Self-hosted metasearch engine protects against tracking from Google, Bing, etc</a> 245 246</div> 247 248 <p><small><div class="author-byline" itemprop="author" itemscope="" itemtype="http://schema.org/Person"> 249By <span itemprop="name" class="author-name-wrapper"> 250 <img class="akashacms-external-links-favicon opengraph-no-promote" src="https://www.google.com/s2/favicons?domain=davidherron.com" style="display: inline-block; padding-right: 2px;" alt="(davidherron.com)"></img> 251 <a href="https://davidherron.com" rel="author" itemprop="url" target="_blank">David Herron</a></span> 252</div>; 253 Date: Wed Jul 13 2022 254 255</small></p> 256 <p>Tags: <span class="taglist"> 257 <a style="padding: 5px;" class="taglink-header p-category" href="../../../tags/docker.html">Docker</a> »»»» 258 259 <a style="padding: 5px;" class="taglink-header p-category" href="../../../tags/self-hosting.html">Self Hosting</a> »»»» 260 </span></p> 261 <p><strong>Are you worried that Google/Bing/etc know too many details about you? These companies collect data about us, to then sell advertising targeting us. This can be avoided using search tools that protect privacy. One, SearXNG, is an open source metasearch engine that guarantees complete anonymity. 262</strong></p> 263 264 265 266 <figure> 267 268 <img src="img/open-source-search.png" class="heropicture"></img> 269 270 271 272 </figure> 273 274 275 276 <div class="mx-auto"> 277 <div id="ezoic-pub-ad-placeholder-102"> 278 </div> 279 </div> 280 281 <p>Traditional centralized search engines like Google or Bing are convenient. Their size means they can pull together a massive index of all websites. They have large teams of software engineers working on improved understanding of good quality content, or search indexing infrastructure, or understanding a variety of human languages, and so on. It's no wonder that the main search engines are so popular, because they're very useful and provide a big value, for free.</p> 282<p>As they say, if you're not paying for the service, then you're the product.</p> 283<p>But, there are a number of problems with the main search engines. Some issues I have are:</p> 284<ul> 285<li>Collecting way too much information, building dossiers on us all, with which they sell us to advertisers</li> 286<li>Every search query gets stashed away by Google, to improve future search results, to present better advertising to you, and so forth.</li> 287<li>Changing search results to downplay organic results, reducing the flow of users from the search engine website to the indexed website (Details: <a href="../../../blog/2019/07/google-taking-more-clicks.html" title="Google taking larger chunk of search traffic, squeezing organic clicks to sites">Google taking larger chunk of search traffic, squeezing organic clicks to sites</a>, <a href="../../../blog/2019/01/google-downplays-organic-results.html" title="Google Ads manipulates Google search results">Google Ads manipulates Google search results</a>)</li> 288</ul> 289<p>You might say "<em>Just use DuckDuckGo</em>", because DuckDuckGo promises it doesn't track who you are. Indeed I've been using DDG for years as my primary search engine. I am happy with the results it gives, and mostly believes that DDG truly does not track what its users are doing.</p> 290<p>But, recently I found an issue in that DDG (and Bing) wasn't indexing a couple of my websites. While working on re
290ctifying that issue I learned that DDG gets (a portion of) its search results data from Bing. DDG uses several search data sources, with Bing being its primary source. As a result folks giving advice on getting your site indexed in DuckDuckGo say to first get it indexed in Bing, and then it will automatically show up in DDG.</p> 291<p>The point for mentioning that is - I did not know that DDG used Bing search data, and is Bing then collecting any data about our searches via DDG? Further, Bing provides search data to several other search engines.</p> 292<figure> 293 294 <img src="./img/searx-alternative.png"></img> 295 296 297</figure> 298 299<h2>SearXNG - the privacy focused alternative to other search engines</h2> 300<p>While looking for alternatives, I came across a self-hosted search engine option made by people who are seriously and deeply interested in personal privacy. Namely - SearXNG - 301 <img class="akashacms-external-links-favicon opengraph-no-promote" src="https://www.google.com/s2/favicons?domain=docs.searxng.org" style="display: inline-block; padding-right: 2px;" alt="(docs.searxng.org)"></img> 302 <a href="https://docs.searxng.org/" target="_blank">https://docs.searxng.org/</a></p> 303<p>SearXNG is derived from another project, SearX. It is what's called a "metasearch engine", meaning it doesn't itself index the Internet, but instead sends search queries out to several other search engines. It then brings those results together, presenting them in a unified search result.</p> 304<figure> 305 306 <img src="./img/what-is-reiki.png"></img> 307 308 309</figure> 310 311<p>This looks like search engine results from other search engines, but there are differences. Notice that each search result listing contains a list of search engines in which the result was found. That's the "metasearch" part of the results. The tabs across the top are also a nice touch. For instance, instead of your video searches being limited to YouTube, they can equally give results from other video websites (Odysee etc).</p> 312<figure> 313 314 <img src="./img/metasearch-wtf.png"></img> 315 316 317</figure> 318 319<h2>What is a Metasearch Engine?</h2> 320<p>I hadn't heard of metasearch engines until finding SearXNG, and I bet you haven't either. To understand why we should install one, we should understand what metasearch engines do, etc. We've just seen an example of a metasearch engine, so lets try to create a definition.</p> 321<p>A metasearch engine is an online information source using the data of one or more other search engines to produce its results.</p> 322<p>In other words, a metasearch engine sends your query to one or more other search engines, aggregates the search results, ranking them based on rankings from the other search engine, presenting the results in its own format.</p> 323<p>There are many examples of commercial metasearch engines. Two prominent ones are <em>DogPile</em> and <em>Metacrawler</em>, and I believe travel sites like <em>Kayak</em> are metasearch engines under the hood.</p> 324<h2>How is SearXNG better at protecting privacy?</h2> 325<p>Okay, you're thinking, SearXNG does search queries on other search engines. How does that differ from DuckDuckGo which also uses other search engines to satisfy search queries. The answer is that SearXNG strictly anonymizes the queries it sends to other search engines.</p> 326<p>The SearXNG documentation says this:</p> 327<ul> 328<li>removal of private data from requests going to search services</li> 329<li>not forwarding anything from a third party services through search services (e.g. advertisement)</li> 330<li>removal of private data from requests going to the result pages</li> 331</ul> 332<p>It does this by not sending cookies to the search engines, and generating a random browser profile for each request. This prevents the search engine from tracking the person who made the query.</p> 333<p>Because it is open source, we can verify these claims for ourselves by studying the source code. DuckDuckGo might be making a similar claim, but we cannot independently verify that they are doing so because DuckDuckGo is not open source.</p> 334<h2>Self-Hosting the SearXNG metasearch engine</h2> 335<p>Learning about how SearXNG protects privacy is what got me interested. After a couple weeks of using it, I'm fairly happy with the results. The only problem is that it often says "timeout" on querying a given search engine.</p> 336<p>The simplest path to self-hosting SearXNG is with Docker. The SearXNG team provides a prebaked Docker Compose file that I used as a starting point.</p> 337<p>See:
338 <img class="akashacms-external-links-favicon opengraph-no-promote" src="https://www.google.com/s2/favicons?domain=github.com" style="display: inline-block; padding-right: 2px;" alt="(github.com)"></img> 339 <a href="https://github.com/searxng/searxng-docker" target="_blank">https://github.com/searxng/searxng-docker</a></p> 340<p>Their preferred deployment uses:</p> 341<ul> 342<li>Caddy - which is a reverse proxy server with bundled support for getting SSL certificates from Lets Encrypt -- I DID NOT USE CADDY IN MY DEPLOYMENT</li> 343<li>SearXNG - the metasearch engine</li> 344<li>REDIS - For caching data</li> 345</ul> 346<p>On my server, I use NGINX Proxy Manager as a reverse proxy that manages SSL certificates from Lets Encrypt. Therefore, I did not include the Caddy portion of their Compose file.</p> 347<p>Start by creating a directory to house some things. In that directory create another directory, <code>searxng</code>, containing a file named <code>settings.yml</code>, containing the following:</p> 348<pre><code class="hljs language-yaml"><span class="hljs-comment"># see https://docs.searxng.org/admin/engines/settings.html#use-default-settings</span> 349<span class="hljs-attr">use_default_settings:</span> <span class="hljs-literal">true</span> 350<span class="hljs-attr">server:</span> 351 <span class="hljs-comment"># base_url is defined in the SEARXNG_BASE_URL environment variable, see .env and docker-compose.yml</span> 352 <span class="hljs-attr">secret_key:</span> <span class="hljs-string">"ultrasecretkey"</span> <span class="hljs-comment"># change this!</span> 353 <span class="hljs-attr">limiter:</span> <span class="hljs-literal">true</span> <span class="hljs-comment"># can be disabled for a private instance</span> 354 <span class="hljs-attr">image_proxy:</span> <span class="hljs-literal">true</span> 355 <span class="hljs-attr">method:</span> <span class="hljs-string">"GET"</span> 356<span class="hljs-attr">search:</span> 357 <span class="hljs-attr">autocomplete:</span> <span class="hljs-string">"duckduckgo"</span> 358<span class="hljs-attr">general:</span> 359 <span class="hljs-attr">instance_name:</span> <span class="hljs-string">'YOUR NAME HERE'</span> 360<span class="hljs-attr">ui:</span> 361 <span class="hljs-attr">static_use_hash:</span> <span class="hljs-literal">true</span> 362<span class="hljs-attr">redis:</span> 363 <span class="hljs-attr">url:</span> <span class="hljs-string">redis://redis:6379/0</span> 364</code></pre> 365<p>This file comes from the <code>searxng-docker</code> repository, with a couple small changes.</p> 366<ul> 367<li>Setting <code>method</code> to <code>GET</code> fixes a behavior issue</li> 368<li>The <code>autocomplete</code> setting lets you specify where autocomplete suggestions come from.</li> 369<li>The <code>instance_name</code> setting is a first step to updating the branding to include your name</li> 370</ul> 371<p>For the super secret key, you can use any string you like. It appears I ran a command like this:</p> 372<pre><code class="hljs"><span class="hljs-meta prompt_">$ </span><span class="language-bash"><span class="hljs-built_in">cat</span> searxng/settings.yml | <span class="hljs-built_in">md5sum</span></span> 373f29bdcafbb3d43e84e85c851459c61e1 374<span class="hljs-meta prompt_"> 375# </span><span class="language-bash">Or, this</span> 376<span class="hljs-meta prompt_">$ </span><span class="language-bash">uuid | <span class="hljs-built_in">md5sum</span></span> 377020a52f6b234e33eb0d4924a7b695683 378</code></pre> 379<p>Whatever way you prefer to generate a randomized string is good.</p> 380<p>Next, create a file named <code>docker-compose.yml</code> containing this:</p> 381<pre><code class="hljs language-yaml"><span class="hljs-attr">version:</span> <span class="hljs-string">'3.7'</span> 382 383<span class="hljs-attr">services:</span> 384 385 <span class="hljs-attr">redis:</span> 386 <span class="hljs-attr">container_name:</span> <span class="hljs-string">redis</span> 387 <span class="hljs-attr">image:</span> <span class="hljs-string">"redis:alpine"</span> 388 <span class="hljs-attr">command:</span> <span class="hljs-string">redis-server</span> <span class="hljs-string">--save</span> <span class="hljs-string">""</span> <span class="hljs-string">--appendonly</span> <span class="hljs-string">"no"</span> 389 <span class="hljs-attr">networks:</span>
390 <span class="hljs-bullet">-</span> <span class="hljs-string">searxng</span> 391 <span class="hljs-attr">tmpfs:</span> 392 <span class="hljs-bullet">-</span> <span class="hljs-string">/var/lib/redis</span> 393 <span class="hljs-attr">cap_drop:</span> 394 <span class="hljs-bullet">-</span> <span class="hljs-string">ALL</span> 395 <span class="hljs-attr">cap_add:</span> 396 <span class="hljs-bullet">-</span> <span class="hljs-string">SETGID</span> 397 <span class="hljs-bullet">-</span> <span class="hljs-string">SETUID</span> 398 <span class="hljs-bullet">-</span> <span class="hljs-string">DAC_OVERRIDE</span> 399 400 <span class="hljs-attr">searxng:</span> 401 <span class="hljs-attr">container_name:</span> <span class="hljs-string">searxng</span> 402 <span class="hljs-attr">image:</span> <span class="hljs-string">searxng/searxng:latest</span> 403 <span class="hljs-attr">networks:</span> 404 <span class="hljs-bullet">-</span> <span class="hljs-string">searxng</span> 405 <span class="hljs-attr">ports:</span> 406 <span class="hljs-bullet">-</span> <span class="hljs-string">"8080:8080"</span> 407 <span class="hljs-attr">volumes:</span> 408 <span class="hljs-bullet">-</span> <span class="hljs-string">./searxng:/etc/searxng:rw</span> 409 <span class="hljs-attr">environment:</span> 410 <span class="hljs-bullet">-</span> <span class="hljs-string">SEARXNG_BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/</span> 411 <span class="hljs-attr">cap_drop:</span> 412 <span class="hljs-bullet">-</span> <span class="hljs-string">ALL</span> 413 <span class="hljs-attr">cap_add:</span> 414 <span class="hljs-bullet">-</span> <span class="hljs-string">CHOWN</span> 415 <span class="hljs-bullet">-</span> <span class="hljs-string">SETGID</span> 416 <span class="hljs-bullet">-</span> <span class="hljs-string">SETUID</span> 417 <span class="hljs-bullet">-</span> <span class="hljs-string">DAC_OVERRIDE</span> 418 <span class="hljs-comment"># logging:</span> 419 <span class="hljs-comment"># driver: "json-file"</span> 420 <span class="hljs-comment"># options:</span> 421 <span class="hljs-comment"># max-size: "1m"</span> 422 <span class="hljs-comment"># max-file: "1"</span> 423 424<span class="hljs-attr">networks:</span> 425 <span class="hljs-attr">searxng:</span> 426 <span class="hljs-attr">external:</span> <span class="hljs-literal">true</span> 427<span class="hljs-comment"># ipam:</span> 428<span class="hljs-comment"># driver: default</span> 429</code></pre> 430<p>This also comes from the <code>searxng-docker</code> repository with some changes. The Caddy service is dropped completely. The <code>searxng</code> network is created externally because of how I manage virtual networks in my Docker installation.</p> 431<p>The volume mount for the <code>searxng</code> directory mounts the configuration file discussed earlier into the container.</p> 432<p>Notice that the <code>SEARXNG_BASE_URL</code> variable is set from an environment variable, <code>SEARXNG_HOSTNAME</code>. To set this as intended, create a file named <code>.env</code> containing:</p> 433<pre><code class="hljs language-bash">SEARXNG_HOSTNAME=searx.DOMAIN-NAME.com 434<span class="hljs-comment"># LETSENCRYPT_EMAIL=<email></span> 435</code></pre> 436<p>This will inform SearXNG what to expect as its domain name. The <code>LETSENCRYPT_EMAIL</code> variable is used with Caddy, if you use that.</p> 437<p>I don't understand why it should use a <code>.env</code> file since these values can be easily coded in the Compose file. And, for that matter, the base URL can be set in the <code>settings.xml</code> in the <code>server.base_url</code> value. That is, in the <code>settings.xml</code> shown above, replace the comment starting with <code>base_url</code> with <code>base_url: "https://searx.DOMAIN"</code>.</p> 438<h2>Deploying the SearXNG metasearch engine</h2> 439<p>That's it for preparation. The files discussed in the previous section must be deployed on a host where you have Docker installed.</p> 440<p>Upon running <code>docker compose up</code> (or <code>docker-compose up</code>), the two services will come up and you'll be able to use the search engine at <code>http://DOMAIN-NAME:8080</code>.</p> 441<p>Notice I said <code>docker compose</code> rather than <code>docker-compose</code>. The latter is the traditional way of launching a Compose file. But, sometime in the last year or so the Docker team ported the functionality of <code>docker-compose</code> into the <code>docker</code> command such that we can now run <code>docker compose</code> (notice - no dash) instead.</p> 442<p>In my case, I'm using NGINX Proxy Manager, and therefore created a proxy host in its configuration. This proxy connects to <code>http://searxng:8080</code> on the backend, and is visible as <code>https://searx.DOMAIN</code> on the front end. Provisioning SSL certificates from Lets Encrypt is trivially easy while setting up the reverse proxy.</p> 443<h2>Using the SearXNG metasearch engine</h2> 444<p>If you've used a certain search engine whose name starts with <code>G</code> you know how to use SearXNG.</p> 445<figure> 446 447 <img src="./img/searx2.png"></img> 448 449 450</figure> 451 452<p>This is the user interface, do I really need to explain what to do?</p> 453<p>An interesting place to look is the settings area, reached via the gear icon in the upper-right corner. You can select the precise search engines being used, and a bunch of other parameters. It's very cool.</p> 454<p>Another cool resource is the documentation, particularly for the
455 <img class="akashacms-external-links-favicon opengraph-no-promote" src="https://www.google.com/s2/favicons?domain=docs.searxng.org" style="display: inline-block; padding-right: 2px;" alt="(docs.searxng.org)"></img> 456 <a href="https://docs.searxng.org/admin/engines/index.html" target="_blank">settings file</a>.</p> 457<p>You can easily customize the list of search engines being used. Because it's open source, you can integrate other search engines by writing code, then configuring them in the settings file.</p> 458<p>There is a tremendous amount of flexibility.</p> 459<p>Just one example covers the case of data you have in your own SQL database. Integrating that data is accomplished something like this:</p> 460<pre><code class="hljs language-yaml"><span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">my_database</span> 461 <span class="hljs-attr">engine:</span> <span class="hljs-string">postgresql</span> 462 <span class="hljs-attr">database:</span> <span class="hljs-string">my_database</span> 463 <span class="hljs-attr">username:</span> <span class="hljs-string">searxng</span> 464 <span class="hljs-attr">password:</span> <span class="hljs-string">password</span> 465 <span class="hljs-attr">query_str:</span> <span class="hljs-string">'SELECT * from my_table WHERE my_column = %(query)s'</span> 466</code></pre> 467<p>In other words, it's possible to use this to build a highly customized search engine.</p> 468<h2>Adding your SearXNG instance as a default search engine in Chrome web browser</h2> 469<p>If you're happy with the results SearXNG gives, you can set up your web browser to use your SearXNG instance. We'll focus on doing this in Google Chrome, but you should be able to do this in other browsers like Firefox.</p> 470<p>Open the <em>Settings</em> area, and navigate to <em>Search engine</em>. You'll see a button for <em>Manage search engines and site search</em>.</p> 471<p>What's required is to add your SearXNG instance as a custom entry in this list, and then to select that as the default search engine.</p> 472<p>There will be an <em>Add</em> button, so click on that.</p> 473<figure> 474 475 <img src="./img/add-search-engine.png"></img> 476 477 478</figure> 479 480<p>For <em>Search engine</em> give it a user-friendly name, and in <em>Shortcut</em> give a shorter user-friendly name. For the URL use: <code>https://searx.DOMAIN/search?q=%s</code> -- Finally, click <em>Add</em>.</p> 481<p>The custom search engine will be added to the list. At the far right hand side is a button you can click which pops up a menu, one of the choices is <em>Make default</em>. Click on that to make it your default search engine.</p> 482<p>With that accomplished, any search performed via the browser location box will go through your SearXNG instance.</p> 483<h2>SearXNG timeouts</h2> 484<p>The only trouble I have is this:</p> 485<figure> 486 487 <img src="./img/searxng-timeout.png"></img> 488 489 490</figure> 491 492<p>Every so often, instead of search results it gives me timeouts like this. In the documentation it appears the timeout threshold can be adjusted. I suspect this is because my instance is hosted at home on a machine sitting behind my DSL router. An instance hosted on a regular web server should have better bandwidth to the Internet, and better response time to search queries.</p> 493<h2>Summary</h2> 494<p>SearXNG is a worthy alternative search engine. It strictly protects you from being tracked by regular search engines, while giving you search results from those search engines.</p> 495<p>I make great use of search engines every day, and use them to find amazing resources. I've used SearXNG exclusively for about 2 weeks, and it has helped me find some amazing things. I didn't mean to overuse the word <em>amazing</em>, but what I mean is that I routinely find useful relavent resources via regular search engines, and so far I've done the same with SearXNG.</p> 496 497 498 499 <div class="author-bio-block"> 500<h3>About the Author(s)</h3> 501 502<a href="https://davidherron.com" target="_blank"><img style="float: right; width: 150px" src="../../../img/headshot-new.jpg"></img></a> 503<strong class="author-bio-block-name"> 504
505 <img class="akashacms-external-links-favicon opengraph-no-promote" src="https://www.google.com/s2/favicons?domain=davidherron.com" style="display: inline-block; padding-right: 2px;" alt="(davidherron.com)"></img> 506 <a href="https://davidherron.com" target="_blank">David Herron</a> 507</strong>: <span class="author-bio-block-bio"> 508David Herron is a writer and software engineer focusing on the wise use of technology. He is especially interested in clean energy technologies like solar power, wind power, and electric cars. David worked for nearly 30 years in Silicon Valley on software ranging from electronic mail systems, to video streaming, to the Java programming language, and has published several books on Node.js programming and electric vehicles. 509</span> 510 511</div> 512 513 514 <div class="btn-group blog-prev-next-wrapper" role="group" aria-label="Next/Previous"> 515<a class="btn btn-outline-primary blog-prev-link" href="../../../linux-sbc/raspberry-pi/automount-usb.html" role="button"> 516<i class="fas fa-angle-double-left fa-3x float-left"></i> 517Using multiple USB disks on a Raspberry Pi OS on RPI 4</a> 518<a class="btn btn-outline-primary blog-next-link" href="dmarc-report.html" role="button"> 519Self-hosted DMARC report viewer to aid your anti-SPAM effort 520<i class="fas fa-angle-double-right fa-3x float-right"></i></a> 521</div> 522 <div class="mx-auto"> 523 <a href="https://www.a2hosting.com/?aid=b39a3569&bid=43e9ce93" rel="" target="_blank"> 524 <img src="../../../imgz/a2hosting/468x60.jpg"></img></a> 525 </div> 526 527<div id="ezoic-pub-ad-placeholder-104"> 528</div> 529 530 <!-- 531Disabled Disqus 532 533<--div id="disqus_thread"></div-- > 534<--script-- > 535 536/** 537* RECOMMENDED CONFIGURATION VARIABLES: EDIT AND UNCOMMENT THE SECTION BELOW TO INSERT DYNAMIC VALUES FROM YOUR PLATFORM OR CMS. 538* LEARN WHY DEFINING THESE VARIABLES IS IMPORTANT: https://disqus.com/admin/universalcode/#configuration-variables*/ 539/* */ 540var disqus_config = function () { 541this.page.url = "<%= rendered_url %>"; // Replace PAGE_URL with your page's canonical URL variable 542this.page.identifier = "<%= document.path %>"; // Replace PAGE_IDENTIFIER with your page's unique identifier variable 543}; 544/* */ 545(function() { // DON'T EDIT BELOW THIS LINE 546var d = document, s = d.createElement('script'); 547s.src = 'https://techsparx-1.disqus.com/embed.js'; 548s.setAttribute('data-timestamp', +new Date()); 549(d.head || d.body).appendChild(s); 550})(); 551</script-- > 552<--noscript>Please enable JavaScript to view the <a href="https://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript-- >-- 553--> 554 555 </div> 556 </article> 557 <section id="right" class="col-md-3"> 558 <!-- Ezoic - sidebar - sidebar --> 559<!-- <div id="ezoic-pub-ad-placeholder-106"> </div> --> 560<!-- End Ezoic - sidebar - sidebar --> 561 562 <div class="card" style="width: 100%; text-align: center;"> 563 <div class="card-body"> 564 <h4 class="card-title">Subscribe via RSS</h4> 565 <a href="../../../feeds.html"> 566 <img src="../../../img/rss_button.gif" align="middle" width="40px"></img> 567 </a> 568 </div> 569</div> 570 571 <!-- <form action="//www.google.com" id="cse-search-box" class="navbar-form navbar-left" role="search"> 572 <input type="hidden" name="cx" value="partner-pub-2401070691062219:9544629023" /> 573 <input type="hidden" name="ie" value="UTF-8" /> 574 <div class="form-group"> 575 <input type="text" class="form-control" placeholder="Search" name="q" /> 576 </div> 577 <button type="submit" name="sa" value="Search" class="btn btn-default">Submit</button> 578
578<script type="text/javascript" src="//www.google.com/coop/cse/brand?form=cse-search-box&lang=en"></script>
578 579</form> 580--> 581 582 583<div> 584 585 586 587<span class="link-to-external-card" style="float: left; width: 100%; padding: 2px; margin: 2px;"> 588 <figure> 589 <a href="../learning.html#MasteringDocker"> 590 <img src="../../../imgz/amazon/417vt2J8hkL.jpg" alt="Mastering Docker - Second Edition: Master this widely used containerization tool" title="Mastering Docker - Second Edition: Master this widely used containerization tool" style="width: 100%;"></img> 591 </a> 592 </figure> 593</span> 594 595<span class="link-to-external-card" style="float: left; width: 100%; padding: 2px; margin: 2px;"> 596 <figure> 597 <a href="../../../nodejs/learning/products.html#UDEMYGruntJSAutomate"> 598 <img src="../../../nodejs/img/nodejs-dark.png" alt="UDEMY: GRUNT js: Automate web development tasks and save your time" title="UDEMY: GRUNT js: Automate web development tasks and save your time" style="width: 100%;"></img> 599 </a> 600 </figure> 601</span> 602 603<span class="link-to-external-card" style="float: left; width: 100%; padding: 2px; margin: 2px;"> 604 <figure> 605 <a href="../../../linux-sbc/raspberry-pi/products.html#OdiySurveilRAMCoolingCoolerHeatsink"> 606 <img src="../../../imgz/amazon/61N7ddVXL6L._SL1024_.jpg" alt="OdiySurveil(TM) 6Pcs Aluminum Chips VGA RAM Cooling Cooler Heatsink for IC MOSFET SCR,DIY for Raspberry Pi" title="OdiySurveil(TM) 6Pcs Aluminum Chips VGA RAM Cooling Cooler Heatsink for IC MOSFET SCR,DIY for Raspberry Pi" style="width: 100%;"></img> 607 </a> 608 </figure> 609</span> 610 611<span class="link-to-external-card" style="float: left; width: 100%; padding: 2px; margin: 2px;"> 612 <figure> 613 <a href="../../../computer-hardware/arduino/products.html#ArduinoUNOOfficialStarterKit"> 614 <img src="https://images-na.ssl-images-amazon.com/images/I/616pbMof2kL._SL1000_.jpg" alt="Arduino Starter Kit - English Official Kit With 170 Page Book" title="Arduino Starter Kit - English Official Kit With 170 Page Book" style="width: 100%;"></img> 615 </a> 616 </figure> 617</span> 618 619<span class="link-to-external-card" style="float: left; width: 100%; padding: 2px; margin: 2px;"> 620 <figure> 621 <a href="../../../lifestyle/geek-gear.html#generic-web-dork"> 622 <img src="../../../imgz/zazzle/generic_web_dork_t_shirt-r82a387581b204b6bbb1fef2528c911b8_k2gr0_1024.jpg" alt="Generic Web Dork T-Shirt" title="Generic Web Dork T-Shirt" style="width: 100%;"></img> 623 </a> 624 </figure> 625</span> 626</div> 627 628 <!-- Ezoic - sidebar - sidebar --> 629<!-- <div id="ezoic-pub-ad-placeholder-106"> </div> --> 630<!-- End Ezoic - sidebar - sidebar --> 631 632 <div> 633 634<a href="https://fave.co/3eo8MDm" rel="" target="_blank"> 635<!-- https://images-na.ssl-images-amazon.com/images/I/41SzsmJa9uL.jpg --> 636<img src="../../../imgs/amazon/41SzsmJa9uL.jpg" align="center" width="100%" style="max-width: 180px; margin-left: auto; margin-right: auto;"></img> 637</a> 638 639<a href="https://www.amazon.com/dp/B072FRZNPD/ref=as_li_ss_tl?ie=UTF8&linkCode=ll1&tag=thereikipage&linkId=d8d0049c1c18a35cb83f2282297119ec" rel="nofollow noskim" target="_blank"> 640 <!-- https://images-na.ssl-images-amazon.com/images/I/510Xc8ieh0L.jpg --> 641 <img src="../../../imgz/amazon/510Xc8ieh0L.jpg" align="center" width="100%" style="max-width: 180px; margin-left: auto; margin-right: auto;"></img> 642</a> 643 644<!-- style="max-width: 150px;" --> 645 646<a href="https://www.amazon.com/Asynchronous-Node-js-JavaScript-Promises-Generators-ebook/dp/B01N2Z4SOR/ref=as_li_ss_tl?s=books&ie=UTF8&qid=1498008506&sr=1-2&linkCode=ll1&tag=thereikipage&linkId=1c85526832f99037c925080682f1db36" rel="nofollow noskim" target="_blank"> 647 <img src="../../../nodejs/img/es-2015-16-17-promises-generators-and-async-await-2-1000.jpg" align="center" width="100%" style="max-width: 180px; margin-left: auto; margin-right: auto;"></img> 648</a> 649 650<a href="https://www.amazon.com/dp/B00WTKDH72/ref=as_li_ss_tl?ie=UTF8&linkCode=ll1&tag=thereikipage&linkId=27adeed4cabfcdcc78a6ce776f8eeae3" rel="nofollow noskim" target="_blank"> 651 <img src="../../../nodejs/img/create-ebook-cover.jpg" align="center" width="100%" style="max-width: 180px; margin-left: auto; margin-right: auto;"></img> 652</a> 653</div> 654 655 <!-- <a target="_blank" href="http://shareasale.com/r.cfm?b=988648&u=101040&m=41312&urllink=&afftrack="><img src="https://i.shareasale.com/image/41312/btg-pc_250x250.jpg" border="0" alt="BlairTG.com Refurbished Computer Sales" /></a> --> 656 <div class="mx-auto"> 657 <div id="ezoic-pub-ad-placeholder-105"> 658 </div> 659 </div> 660 </section> 661</div> 662 663<div class="row"> 664 <!-- tag-cloud class="col-md-12"/ --> 665</div> 666 667<footer class="row" style="text-align: center; height: 100px;"> 668<span class="col-md-8"> 669 <a href="../../../about.html">About TechSparx</a> 670 671 672<!-- span class="col-md-3">< %- partial('google-voice.html') %></span --> 673</span></footer> 674 675</div> <!-- container --> 676 677 678<!-- JavaScript at the bottom for fast page loading --> 679
680<script src="../../../vendor/jquery/jquery.min.js"></script>
680<script src="../../../vendor/popper.js/umd/popper.min.js"></script>
680<script src="../../../vendor/bootstrap/js/bootstrap.min.js"></script>
680<script src="../../../vendor/highlight.js/lib/highlight.js"></script>
680<script>hljs.initHighlightingOnLoad();</script>
680 681<!--
681<script>
vendor: 345 bytes, lines 681-687
681 682 (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ 683 (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), 684 m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) 685 })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); 686 687 ga('create', '
687UA-96446326-1
vendor: 39 bytes, lines 687-689
687', 'auto'); 688 ga('send', 'pageview'); 689
690</script>
690 691--> 692<!-- Go to www.addthis.com/dashboard to customize your tools -->
692<script type="text/javascript" src="//s7.addthis.com/js/300/addthis_widget.js#pubid=ra-594227a34ce36d45"></script>
692 693 694
695<script type="text/javascript" src="//s.skimresources.com/js/27810X1162539.skimlinks.js"></script>
695 696</body> 697</html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.