gatsby-transformer-remark excerpt is not available in @andrew-codes/gatsby-plugin-elasticlunr-search

[JacobTheEvans]

Summary

Im trying to use the gatsby-plugin-elasticlunr-search to search through markdown files processed by the gatsby-transformer-remark plugin. When building my index all keys resolve but the excerpt key. Its value is always a blank string. But when I query the markdown files in page the experts have been generated. Are the excerpts generated after the gatsby-config.js is ran or am I running into a valid error?

Relevant information

Example Markdown Page:

---
title: "Bacnet Device"
date: "2018-06-18"
author: "Jacob Evans"
tags: ["bacnet", "device"]
video: "DjXG2Ol7K2o"
---

[Bacnet](http://www.bacnet.org/) is awesome.

BACnet is a communications protocol for Building Automation and Control (BAC) networks that leverage the ASHRAE, ANSI, and ISO 16484-5 standard protocol.

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur commodo lectus eget lacus vulputate tempus. Donec lacinia varius eros eu dictum. Morbi non tortor ipsum. Morbi tempus est tellus, quis ultrices urna tempor at. Cras sed eros euismod, cursus metus sed, placerat velit. Aenean in vestibulum quam, non tincidunt nulla. Donec pretium ac magna et hendrerit. Phasellus justo justo, tincidunt id purus eget, sollicitudin pulvinar tellus. Duis eget massa id dui lacinia tempor. Nam elementum nisi non nisi interdum interdum. Maecenas mi velit, suscipit nec lectus ut, pellentesque eleifend nisi. Aliquam a aliquet urna. Mauris volutpat tristique ex sit amet interdum. Duis vitae nibh nec erat egestas vehicula dignissim vel lectus. Morbi fermentum massa in mi auctor fermentum. Nullam rutrum pretium consequat.

gatsby-config.js:

module.exports = {
  siteMetadata: {
    title: 'Example Project',
    slogan: 'Some Slogan',
    company: 'Company Name',
    companyUrl: 'jacobtheevans.com',
  },
  plugins: [
    {
      resolve: `gatsby-source-filesystem`,
      options: {
        name: 'src',
        path: `${__dirname}/src/`
      }
    },
    {
      resolve: `gatsby-plugin-node-fields`,
      options: {
        descriptors: [
          {
            predicate: (node) => node.frontmatter,
            fields: [
              {
                name: 'video',
                getter: node => node.frontmatter.video,
                defaultValue: ''
              }
            ]
          }
        ]
      }
    },
    {
      resolve: `gatsby-plugin-google-fonts`,
      options: {
        fonts: [
          `roboto`,
          'roboto-slab'
        ]
      }
    },
    'gatsby-plugin-react-helmet',
    'gatsby-transformer-remark',
    {
      resolve: `@andrew-codes/gatsby-plugin-elasticlunr-search`,
      options: {
        fields: ['title', 'tags', 'slug', 'author', 'date', 'excerpt'],
        resolvers: {
          MarkdownRemark: {
            excerpt: node => {
              console.log(node) // prints out blank string
              return node.excerpt
            },
            title: node => node.frontmatter.title,
            tags: node => node.frontmatter.tags,
            slug: node => node.fields.slug,
            author: node => node.frontmatter.author,
            date: node => node.frontmatter.date
          }
        }
      }
    }
  ]
}

Environment

  System:
    OS: Linux 4.16 Fedora 27 (Workstation Edition) 27 (Workstation Edition)
    CPU: x64 Intel(R) Core(TM) i7-7500U CPU @ 2.70GHz
    Shell: 4.4.23 - /bin/bash
  Binaries:
    Node: 8.11.2 - /usr/bin/node
    Yarn: 1.7.0 - /usr/bin/yarn
    npm: 5.6.0 - /usr/bin/npm
  Browsers:
    Firefox: 60.0.2
  npmPackages:
    gatsby: ^1.9.247 => 1.9.273 
    gatsby-link: ^1.6.40 => 1.6.45 
    gatsby-plugin-google-fonts: ^0.0.4 => 0.0.4 
    gatsby-plugin-node-fields: ^0.0.6 => 0.0.6 
    gatsby-plugin-react-helmet: ^2.0.10 => 2.0.11 
    gatsby-source-filesystem: ^1.5.39 => 1.5.39 
    gatsby-transformer-remark: ^1.7.44 => 1.7.44 

File contents (if changed)

package.json:

{
  "name": "example-project",
  "description": "Learning and trying",
  "version": "1.0.0",
  "author": "Jacob Evans",
  "dependencies": {
    "@andrew-codes/gatsby-plugin-elasticlunr-search": "^1.0.4",
    "elasticlunr": "0.9.5",
    "gatsby": "^1.9.247",
    "gatsby-link": "^1.6.40",
    "gatsby-plugin-google-fonts": "^0.0.4",
    "gatsby-plugin-node-fields": "^0.0.6",
    "gatsby-plugin-react-helmet": "^2.0.10",
    "gatsby-source-filesystem": "^1.5.39",
    "gatsby-transformer-remark": "^1.7.44",
    "react-fontawesome": "^1.6.1",
    "react-helmet": "^5.2.0",
    "react-redux": "^5.0.7",
    "react-youtube": "^7.6.0",
    "redux": "^4.0.0",
    "styled-components": "^3.3.2"
  },
  "license": "MIT",
  "scripts": {
    "build": "gatsby build",
    "develop": "gatsby develop",
    "format": "prettier --write 'src/**/*.js'",
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "devDependencies": {
    "prettier": "^1.12.0"
  },
  "repository": {
    "type": "git",
    "url": "https://github.com/gatsbyjs/gatsby-starter-default"
  }
}

gatsby-node.js:

const { createFilePath } = require(`gatsby-source-filesystem`)
const path = require('path')

exports.onCreateNode = ({ node, getNode, boundActionCreators }) => {
  const { createNodeField } = boundActionCreators
  if (node.internal.type === `MarkdownRemark`) {
    const value = createFilePath({ node, getNode, basePath: 'pages' })
    createNodeField({
      node,
      value,
      name: 'slug'
    })
  }
}

exports.createPages = async ({ graphql, boundActionCreators }) => {
  const { createPage } = boundActionCreators
  const result = await graphql(`
  {
    allMarkdownRemark {
      edges {
        node {
          fields {
            slug
          }
        }
      }
    }
  }
  `)
  if (result.errors) throw result.errors
  for (let edge of result.data.allMarkdownRemark.edges) {
    const { node } = edge
    createPage({
      path: node.fields.slug,
      component: path.resolve('./src/templates/blog-post.js'),
      context: {
        slug: node.fields.slug
      }
    })
  }
}

gatsby-browser.js:

import React from 'react'
import { Router } from 'react-router-dom'
import { Provider } from 'react-redux'

import store from './src/redux'

export const replaceRouterComponent = ({ history }) => {
  const ConnectedRouterWrapper = ({ children }) => (
    <Provider store={store}>
      <Router history={history}>{children}</Router>
    </Provider>
  )
  return ConnectedRouterWrapper
}

gatsby-ssr.js:

import React from 'react'
import { Provider } from 'react-redux'
import { renderToString } from 'react-dom/server'

import { store } from './src/redux'

export const replaceRenderer = ({ bodyComponent, replaceBodyHTMLString }) => {
  const ConnectedBody = () => (
    <Provider store={store}>
      {bodyComponent}
    </Provider>
    )
  replaceBodyHTMLString(renderToString(<ConnectedBody />))
}

Please let me know if there is any other information I can provide to help debug this.

[Mrtenz]

gatsby-plugin-elasticlunr-search creates the search index when onCreateNode is called, before the GraphQL schema is built. gatsby-transformer-remark uses setFieldsOnGraphQLNodeType to parse the markdown to HTML, which is called when the schema is being built. When the MarkdownRemark node is initially created, only the frontmatter is available.

Personally I solved it by using lunr.js and build the search index manually. Not an ideal solution, but it's a bit more flexible. I might make a plugin for it...

[JacobTheEvans]

I think a plugin would be welcome in the meantime I will take your advice and build the search index manually.

[JacobTheEvans]

Just in case anyone body wants an easier solution in the future to deal with this. I looked into the source code of the gatsby-transformer-remark and just ported the way there were making excerpts into the actual resolver like so.

gatsby-config.js:

const remark = require('remark')
const visit = require('unist-util-visit')

module.exports = {
  siteMetadata: {
    title: 'Example Project',
    slogan: 'Your slogan',
  },
  plugins: [
    {
      resolve: `gatsby-source-filesystem`,
      options: {
        name: 'src',
        path: `${__dirname}/src/`
      }
    },
    {
      resolve: `gatsby-plugin-node-fields`,
      options: {
        descriptors: [
          {
            predicate: (node) => node.frontmatter,
            fields: [
              {
                name: 'video',
                getter: node => node.frontmatter.video,
                defaultValue: ''
              }
            ]
          }
        ]
      }
    },
    {
      resolve: `gatsby-plugin-google-fonts`,
      options: {
        fonts: [
          `roboto`,
          'roboto-slab'
        ]
      }
    },
    'gatsby-plugin-react-helmet',
    'gatsby-transformer-remark',
    {
      resolve: `@andrew-codes/gatsby-plugin-elasticlunr-search`,
      options: {
        fields: ['title', 'tags', 'slug', 'author', 'date', 'excerpt'],
        resolvers: {
          MarkdownRemark: {
            excerpt: node => {
              const excerptLength = 136 // Hard coded excerpt length
              let excerpt = ''
              const tree = remark().parse(node.rawMarkdownBody)
              visit(tree, 'text', (node) => {
                excerpt += node.value
              })
              return excerpt.slice(0, excerptLength) + '...'
            },
            title: node => node.frontmatter.title,
            tags: node => node.frontmatter.tags,
            slug: node => node.fields.slug,
            author: node => node.frontmatter.author,
            date: node => node.frontmatter.date
          }
        }
      }
    }
  ]
}

[DanielRuf]

Is this just in v2 possible?

[krismorf]

@Mrtenz @JacobTheEvans
How can I build my own search index? I'm using a local version of gatsby-plugin-elasticlunr-search and I'm stuck on getting the images to work. Can you share your process?

[carsonkk]

@JacobTheEvans method worked great. I wanted the timeToRead property so I also went through and traced the call stack for it in gatsby-transformer-remark and came up with this:

const _ = require(`lodash`)
const remark = require('remark')
const sanitizeHTML = require(`sanitize-html`)
const toHAST = require(`mdast-util-to-hast`)
const hastToHTML = require(`hast-util-to-html`)
const visit = require('unist-util-visit')

module.exports = {
  siteMetadata: {
    ...
  },
  plugins: [
    ...
    {
      resolve: `@gatsby-contrib/gatsby-plugin-elasticlunr-search`,
      options: {
        fields: [
          'title',
          'name',
          'slug',
          'date',
          'tags',
          'excerpt',
          'timeToRead',
        ],
        resolvers: {
          MarkdownRemark: {
            title: node => node.frontmatter.title,
            name: node => node.frontmatter.name,
            slug: node => node.fields.slug,
            date: node => node.fields.date,
            tags: node => node.frontmatter.tags,
            excerpt: node => {
              const length = 136
              const tree = remark().parse(node.rawMarkdownBody)
              let excerpt = ''
              visit(tree, 'text', (node) => {
                excerpt += node.value
              })
              return excerpt.slice(0, length) + '...'
            },
            timeToRead: node => {
              const avgWPM = 265
              const tree = remark().parse(node.internal.content)
              const htmlAst = toHAST(tree, { allowDangerousHTML: true })
              const html = hastToHTML(htmlAst, { allowDangerousHTML: true })
              const pureText = sanitizeHTML(html, { allowTags: [] })
              const wordCount = _.words(pureText).length
              let timeToRead = Math.ceil(wordCount / avgWPM)
              if (timeToRead === 0) {
                timeToRead = 1
              }
              return timeToRead
            },
          }
        }
      }
    }
  ],
}

In the original source it uses Math.round(), not Math.ceil(), however this seemed to give me more off-by-1 values than not, so if you have issues with this try logging the word count for each of your articles and checking what the result/avgWPM comes out to be.

[Tomekmularczyk]

Little more elegant solution for creating excerpt from raw markdown:

const remark = require("remark");
const stripMarkdown = require("strip-markdown");

   {
      resolve: "@gatsby-contrib/gatsby-plugin-elasticlunr-search",
      options: {
        fields: ["title", "date", "excerpt"],
        resolvers: {
          MarkdownRemark: {
            title: node => node.frontmatter.title,
            date: node => node.frontmatter.date,
            excerpt: node => {
              const text = remark()
                .use(stripMarkdown)
                .processSync(node.rawMarkdownBody);

              const excerptLength = 140; // Hard coded excerpt length
              return String(text).substring(0, excerptLength) + "...";
            },
          }
        }
      }
   }

[siawyoung]

Man, I'm facing the exact same issue, trying to replicate functionality for formatting dates and excerpts etc. This works great, thanks a lot!