knowledge/scripts/sync-source.sh at main · elizaOS/knowledge · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
#!/bin/bash
# Sync individual data sources
#
# Usage:
#   ./scripts/sync-source.sh daily-silk
#   ./scripts/sync-source.sh ai-news
#   ./scripts/sync-source.sh github
#   ./scripts/sync-source.sh docs
#   ./scripts/sync-source.sh all

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(dirname "$SCRIPT_DIR")"
cd "$REPO_ROOT"

sync_daily_silk() {
    echo "Syncing daily-silk..."
    rm -rf temp-daily-silk
    git clone --depth 1 https://github.com/madjin/daily-silk.git temp-daily-silk
    mkdir -p daily-silk
    find temp-daily-silk/data -name "*.md" | tar -cf - -T - | tar -xf - --strip-components=2 -C daily-silk/
    rm -rf temp-daily-silk
    echo "Done: daily-silk"
}

sync_ai_news() {
    echo "Syncing ai-news..."
    rm -rf temp-ai-news
    git clone --depth 1 --branch gh-pages https://github.com/M3-org/ai-news.git temp-ai-news
    mkdir -p ai-news/elizaos ai-news/hyperfy
    rsync -av --delete temp-ai-news/elizaos/json/ ai-news/elizaos/json/
    rsync -av --delete temp-ai-news/elizaos/md/ ai-news/elizaos/md/
    rsync -av --delete temp-ai-news/elizaos/discord/ ai-news/elizaos/discord/
    [ -d temp-ai-news/hyperfy ] && rsync -av --delete temp-ai-news/hyperfy/ ai-news/hyperfy/
    rm -rf temp-ai-news
    echo "Done: ai-news"
}

sync_github() {
    echo "Syncing github..."
    rm -rf temp-github
    git clone --depth 1 --branch _data https://github.com/elizaOS/elizaos.github.io.git temp-github
    mkdir -p github/summaries github/api github/contributors
    rsync -av --delete temp-github/data/elizaos_eliza/ github/ --exclude='summaries' --exclude='api' --exclude='contributors'
    rsync -av --delete temp-github/data/summaries/ github/summaries/
    rsync -av --delete temp-github/data/api/ github/api/
    rsync -av --delete temp-github/data/contributors/ github/contributors/
    rm -rf temp-github
    echo "Done: github"
}

sync_docs() {
    echo "Syncing docs..."
    rm -rf temp-docs
    git clone --depth 1 --branch develop https://github.com/elizaOS/eliza.git temp-docs
    mkdir -p docs
    rsync -av --delete temp-docs/docs/ docs/
    rm -rf temp-docs
    echo "Done: docs"
}

case "${1:-}" in
    daily-silk)
        sync_daily_silk
        ;;
    ai-news)
        sync_ai_news
        ;;
    github)
        sync_github
        ;;
    docs)
        sync_docs
        ;;
    all)
        sync_daily_silk
        sync_ai_news
        sync_github
        sync_docs
        ;;
    *)
        echo "Usage: $0 {daily-silk|ai-news|github|docs|all}"
        exit 1
        ;;
esac