Added go-export-page-content example

2022-04-10 16:49:21 +01:00 · 2022-04-10 16:49:21 +01:00 · 29922fcfeb
commit 29922fcfeb
parent d22d2eb86f
8 changed files with 403 additions and 0 deletions
--- a/go-export-page-content/.gitignore
+++ b/go-export-page-content/.gitignore
@ -0,0 +1,6 @@
+bookstack-export/ 
+page-export/ 
+bookstack-export 
+bookstack-export.exe 
+.idea/ 
+bin/ 
--- a/go-export-page-content/api.go
+++ b/go-export-page-content/api.go
@ -0,0 +1,140 @@
+package main 
+  
+import ( 
+"crypto/tls" 
+"encoding/json" 
+"fmt" 
+"io/ioutil" 
+"net/http" 
+"net/url" 
+"strconv" 
+"strings" 
+) 
+  
+type BookStackApi struct { 
+BaseURL string 
+TokenID string 
+TokenSecret string 
+} 
+  
+func NewBookStackApi(baseUrl string, tokenId string, tokenSecret string) *BookStackApi { 
+api := &BookStackApi{ 
+BaseURL: baseUrl, 
+TokenID: tokenId, 
+TokenSecret: tokenSecret, 
+} 
+  
+return api 
+} 
+  
+func (bs BookStackApi) authHeader() string { 
+return fmt.Sprintf("Token %s:%s", bs.TokenID, bs.TokenSecret) 
+} 
+  
+func (bs BookStackApi) getRequest(method string, urlPath string, data map[string]string) *http.Request { 
+method = strings.ToUpper(method) 
+completeUrlStr := fmt.Sprintf("%s/api/%s", strings.TrimRight(bs.BaseURL, "/"), strings.TrimLeft(urlPath, "/")) 
+  
+queryValues := url.Values{} 
+for k, v := range data { 
+queryValues.Add(k, v) 
+} 
+encodedData := queryValues.Encode() 
+  
+r, err := http.NewRequest(method, completeUrlStr, strings.NewReader(encodedData)) 
+if err != nil { 
+panic(err) 
+} 
+  
+r.Header.Add("Authorization", bs.authHeader()) 
+  
+if method != "GET" && method != "HEAD" { 
+r.Header.Add("Content-Type", "application/x-www-form-urlencoded") 
+r.Header.Add("Content-Length", strconv.Itoa(len(encodedData))) 
+} else { 
+r.URL.RawQuery = encodedData 
+} 
+  
+return r 
+} 
+  
+func (bs BookStackApi) doRequest(method string, urlPath string, data map[string]string) []byte { 
+client := &http.Client{ 
+Transport: &http.Transport{ 
+TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 
+}, 
+} 
+r := bs.getRequest(method, urlPath, data) 
+res, err := client.Do(r) 
+if err != nil { 
+panic(err) 
+} 
+  
+defer res.Body.Close() 
+  
+body, err := ioutil.ReadAll(res.Body) 
+if err != nil { 
+panic(err) 
+} 
+  
+return body 
+} 
+  
+func (bs BookStackApi) getFromListResponse(responseData []byte, models any) ListResponse { 
+var response ListResponse 
+  
+if err := json.Unmarshal(responseData, &response); err != nil { 
+panic(err) 
+} 
+  
+if err := json.Unmarshal(response.Data, models); err != nil { 
+panic(err) 
+} 
+  
+return response 
+} 
+  
+func (bs BookStackApi) GetBooks(count int, page int) ([]Book, int) { 
+var books []Book 
+  
+data := bs.doRequest("GET", "/books", getPagingParams(count, page)) 
+response := bs.getFromListResponse(data, &books) 
+  
+return books, response.Total 
+} 
+  
+func (bs BookStackApi) GetChapters(count int, page int) ([]Chapter, int) { 
+var chapters []Chapter 
+  
+data := bs.doRequest("GET", "/chapters", getPagingParams(count, page)) 
+response := bs.getFromListResponse(data, &chapters) 
+  
+return chapters, response.Total 
+} 
+  
+func (bs BookStackApi) GetPages(count int, page int) ([]Page, int) { 
+var pages []Page 
+  
+data := bs.doRequest("GET", "/pages", getPagingParams(count, page)) 
+response := bs.getFromListResponse(data, &pages) 
+  
+return pages, response.Total 
+} 
+  
+func (bs BookStackApi) GetPage(id int) Page { 
+var page Page 
+  
+data := bs.doRequest("GET", fmt.Sprintf("/pages/%d", id), nil) 
+if err := json.Unmarshal(data, &page); err != nil { 
+panic(err) 
+} 
+  
+return page 
+} 
+  
+func getPagingParams(count int, page int) map[string]string { 
+return map[string]string{ 
+"count": strconv.Itoa(count), 
+"offset": strconv.Itoa(count * (page - 1)), 
+} 
+} 
--- a/go-export-page-content/build.sh
+++ b/go-export-page-content/build.sh
@ -0,0 +1,7 @@
+#!/bin/sh 
+  
+GOOS=windows GOARCH=amd64 go build -ldflags "-s -w" -o bin/bookstack-export.exe 
+GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o bin/bookstack-export 
+GOOS=darwin GOARCH=amd64 go build -ldflags "-s -w" -o bin/bookstack-export-macos 
+  
+upx bin/* 
--- a/go-export-page-content/content-map-funcs.go
+++ b/go-export-page-content/content-map-funcs.go
@ -0,0 +1,68 @@
+package main 
+  
+import ( 
+"time" 
+) 
+  
+func getBookMap(api *BookStackApi) map[int]Book { 
+var books []Book 
+var byId = make(map[int]Book) 
+  
+page := 1 
+hasMoreBooks := true 
+for hasMoreBooks { 
+time.Sleep(time.Second / 2) 
+newBooks, _ := api.GetBooks(200, page) 
+hasMoreBooks = len(newBooks) == 200 
+page++ 
+books = append(books, newBooks...) 
+} 
+  
+for _, book := range books { 
+byId[book.Id] = book 
+} 
+  
+return byId 
+} 
+  
+func getChapterMap(api *BookStackApi) map[int]Chapter { 
+var chapters []Chapter 
+var byId = make(map[int]Chapter) 
+  
+page := 1 
+hasMoreChapters := true 
+for hasMoreChapters { 
+time.Sleep(time.Second / 2) 
+newChapters, _ := api.GetChapters(200, page) 
+hasMoreChapters = len(newChapters) == 200 
+page++ 
+chapters = append(chapters, newChapters...) 
+} 
+  
+for _, chapter := range chapters { 
+byId[chapter.Id] = chapter 
+} 
+  
+return byId 
+} 
+  
+func getPageMap(api *BookStackApi) map[int]Page { 
+var pages []Page 
+var byId = make(map[int]Page) 
+  
+page := 1 
+hasMorePages := true 
+for hasMorePages { 
+time.Sleep(time.Second / 2) 
+newPages, _ := api.GetPages(200, page) 
+hasMorePages = len(newPages) == 200 
+page++ 
+pages = append(pages, newPages...) 
+} 
+  
+for _, page := range pages { 
+byId[page.Id] = page 
+} 
+  
+return byId 
+} 
--- a/go-export-page-content/export.go
+++ b/go-export-page-content/export.go
@ -0,0 +1,87 @@
+package main 
+  
+import ( 
+"flag" 
+"fmt" 
+"os" 
+"path/filepath" 
+"time" 
+) 
+  
+func main() { 
+  
+baseUrlPtr := flag.String("baseurl", "", "The base URL of your BookStack instance") 
+tokenId := flag.String("tokenid", "", "Your BookStack API Token ID") 
+tokenSecret := flag.String("tokensecret", "", "Your BookStack API Token Secret") 
+exportDir := flag.String("exportdir", "./page-export", "The directory to store exported data") 
+  
+flag.Parse() 
+  
+if *baseUrlPtr == "" || *tokenId == "" || *tokenSecret == "" { 
+panic("baseurl, tokenid and tokensecret arguments are required") 
+} 
+  
+api := NewBookStackApi(*baseUrlPtr, *tokenId, *tokenSecret) 
+  
+// Grab all content from BookStack 
+fmt.Println("Fetching books...") 
+bookIdMap := getBookMap(api) 
+fmt.Printf("Fetched %d books\n", len(bookIdMap)) 
+fmt.Println("Fetching chapters...") 
+chapterIdMap := getChapterMap(api) 
+fmt.Printf("Fetched %d chapters\n", len(chapterIdMap)) 
+fmt.Println("Fetching pages...") 
+pageIdMap := getPageMap(api) 
+fmt.Printf("Fetched %d pages\n", len(pageIdMap)) 
+  
+// Track progress when going through our pages 
+pageCount := len(pageIdMap) 
+currentCount := 1 
+  
+// Cycle through each of our fetches pages 
+for _, p := range pageIdMap { 
+fmt.Printf("Exporting page %d/%d [%s]\n", currentCount, pageCount, p.Name) 
+// Get the full page content 
+fullPage := api.GetPage(p.Id) 
+  
+// Work out a book+chapter relative path 
+book := bookIdMap[fullPage.BookId] 
+path := book.Slug 
+if chapter, ok := chapterIdMap[fullPage.ChapterId]; ok { 
+path = "/" + chapter.Slug 
+} 
+  
+// Get the html, or markdown, content from our page along with the file name 
+// based upon the page slug 
+content := fullPage.Html 
+fName := fullPage.Slug + ".html" 
+if fullPage.Markdown != "" { 
+content = fullPage.Markdown 
+fName = fullPage.Slug + ".md" 
+} 
+  
+// Create our directory path 
+absExportPath, err := filepath.Abs(*exportDir) 
+if err != nil { 
+panic(err) 
+} 
+  
+absPath := filepath.Join(absExportPath, path) 
+err = os.MkdirAll(absPath, 0744) 
+if err != nil { 
+panic(err) 
+} 
+  
+// Write the content to the filesystem 
+fPath := filepath.Join(absPath, fName) 
+err = os.WriteFile(fPath, []byte(content), 0644) 
+if err != nil { 
+panic(err) 
+} 
+  
+// Wait to avoid hitting rate limits 
+time.Sleep(time.Second / 4) 
+currentCount++ 
+} 
+  
+} 
--- a/go-export-page-content/go.mod
+++ b/go-export-page-content/go.mod
@ -0,0 +1,3 @@
+module bookstack-export 
+  
+go 1.18 
--- a/go-export-page-content/models.go
+++ b/go-export-page-content/models.go
@ -0,0 +1,54 @@
+package main 
+  
+import ( 
+"encoding/json" 
+"time" 
+) 
+  
+type ListResponse struct { 
+Data json.RawMessage `json:"data"` 
+Total int `json:"total"` 
+} 
+  
+type Book struct { 
+Id int `json:"id"` 
+Name string `json:"name"` 
+Slug string `json:"slug"` 
+Description string `json:"description"` 
+CreatedAt time.Time `json:"created_at"` 
+UpdatedAt time.Time `json:"updated_at"` 
+CreatedBy int `json:"created_by"` 
+UpdatedBy int `json:"updated_by"` 
+OwnedBy int `json:"owned_by"` 
+ImageId int `json:"image_id"` 
+} 
+  
+type Chapter struct { 
+Id int `json:"id"` 
+BookId int `json:"book_id"` 
+Name string `json:"name"` 
+Slug string `json:"slug"` 
+Description string `json:"description"` 
+Priority int `json:"priority"` 
+CreatedAt string `json:"created_at"` 
+UpdatedAt time.Time `json:"updated_at"` 
+CreatedBy int `json:"created_by"` 
+UpdatedBy int `json:"updated_by"` 
+OwnedBy int `json:"owned_by"` 
+} 
+  
+type Page struct { 
+Id int `json:"id"` 
+BookId int `json:"book_id"` 
+ChapterId int `json:"chapter_id"` 
+Name string `json:"name"` 
+Slug string `json:"slug"` 
+Html string `json:"html"` 
+Priority int `json:"priority"` 
+CreatedAt time.Time `json:"created_at"` 
+UpdatedAt time.Time `json:"updated_at"` 
+Draft bool `json:"draft"` 
+Markdown string `json:"markdown"` 
+RevisionCount int `json:"revision_count"` 
+Template bool `json:"template"` 
+} 
--- a/go-export-page-content/readme.md
+++ b/go-export-page-content/readme.md
@ -0,0 +1,38 @@
+# Export Page Content 
+  
+This project, written in Go, will export all page content in its original written form (HTML or Markdown). 
+Content will be written into a directory structure that mirrors the page's location within the BookStack content hierarchy (Book > Chapter > Page). 
+  
+Note: This is only provided as an example. The project lacks full error handling and also disables HTTPS verification for easier use with self-signed certificates. 
+  
+## Requirements 
+  
+[Go](https://go.dev/) is required to build this project. 
+This project was built and tested using Go 1.18. 
+  
+You will need your BookStack API credentials at the ready. 
+  
+## Building 
+  
+```bash 
+# Clone down the api-scripts repo and enter this directory 
+git clone https://github.com/BookStackApp/api-scripts.git 
+cd api-scripts/go-export-page-content 
+go build 
+``` 
+  
+This will output a `bookstack-export` executable file. 
+  
+A `build.sh` script is provided to build compressed binaries for multiple platforms. 
+This requires `upx` for the compression element. 
+  
+## Running 
+  
+You can run the project by running the executable file like so: 
+  
+```bash 
+./bookstack-export --baseurl=https://bookstack.example.com --tokenid=abc123 --tokensecret=def456 
+``` 
+  
+By default, this will output to a `page-export` directory within the current working directory. 
+You can define the output directory via a `--exportdir=<dir>` option.