Add local source ingest and podcasts
This commit is contained in:
Krishna Kumar
@@ -56,9 +56,13 @@ struct ClaudeService: Sendable {
|
||||
|
||||
// MARK: - Podcast
|
||||
|
||||
func generatePodcast(url: String) async throws -> String {
|
||||
func generatePodcast(url: String, title: String? = nil, sourceText: String? = nil, sourceKind: String? = nil) async throws -> String {
|
||||
struct Response: Decodable { let job_id: String }
|
||||
let data = try await post("/v1/podcast/generate", body: ["url": url])
|
||||
var body: [String: Any] = ["url": url]
|
||||
if let title, !title.isEmpty { body["title"] = title }
|
||||
if let sourceText, !sourceText.isEmpty { body["text"] = sourceText }
|
||||
if let sourceKind, !sourceKind.isEmpty { body["source_kind"] = sourceKind }
|
||||
let data = try await post("/v1/podcast/generate", body: body)
|
||||
return try JSONDecoder().decode(Response.self, from: data).job_id
|
||||
}
|
||||
|
||||
|
||||
203
Marks/Services/IngestedSourceStore.swift
Normal file
203
Marks/Services/IngestedSourceStore.swift
Normal file
@@ -0,0 +1,203 @@
|
||||
import Foundation
|
||||
import Observation
|
||||
import PDFKit
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
@MainActor
|
||||
@Observable
|
||||
final class IngestedSourceLibrary {
|
||||
private(set) var sources: [IngestedSource] = []
|
||||
var error: String?
|
||||
|
||||
private let store: IngestedSourceStore
|
||||
|
||||
init(store: IngestedSourceStore = IngestedSourceStore()) {
|
||||
self.store = store
|
||||
}
|
||||
|
||||
func load() {
|
||||
do {
|
||||
sources = try store.load()
|
||||
Task { await SourceSpotlightIndexer.index(sources) }
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
|
||||
func addText(title: String, body: String, tags: [String]) {
|
||||
let source = IngestedSource(
|
||||
kind: .text,
|
||||
title: title,
|
||||
bodyText: body.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
tags: tags
|
||||
)
|
||||
upsert(source)
|
||||
}
|
||||
|
||||
func importFile(from url: URL, tags: [String]) {
|
||||
do {
|
||||
let source = try store.importFile(from: url, tags: tags)
|
||||
upsert(source)
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
|
||||
func delete(_ source: IngestedSource) {
|
||||
do {
|
||||
try store.delete(source)
|
||||
sources.removeAll { $0.id == source.id }
|
||||
Task { await SourceSpotlightIndexer.remove(ids: [source.id]) }
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
|
||||
func fileURL(for source: IngestedSource) -> URL? {
|
||||
store.fileURL(for: source)
|
||||
}
|
||||
|
||||
func search(_ query: String) -> [IngestedSource] {
|
||||
let q = query.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||
guard !q.isEmpty else { return sources }
|
||||
return sources.filter { source in
|
||||
source.displayTitle.lowercased().contains(q) ||
|
||||
source.bodyText.lowercased().contains(q) ||
|
||||
source.tags.contains { $0.lowercased().contains(q) } ||
|
||||
(source.originalFilename ?? "").lowercased().contains(q)
|
||||
}
|
||||
}
|
||||
|
||||
private func upsert(_ source: IngestedSource) {
|
||||
sources.removeAll { $0.id == source.id }
|
||||
sources.insert(source, at: 0)
|
||||
do {
|
||||
try store.save(sources)
|
||||
Task { await SourceSpotlightIndexer.index([source]) }
|
||||
} catch {
|
||||
self.error = error.localizedDescription
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct IngestedSourceStore {
|
||||
private let rootURL: URL
|
||||
private let metadataURL: URL
|
||||
private let filesURL: URL
|
||||
|
||||
init(rootURL: URL? = nil) {
|
||||
let base = rootURL ?? FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0]
|
||||
self.rootURL = base.appendingPathComponent("Sources", isDirectory: true)
|
||||
self.metadataURL = self.rootURL.appendingPathComponent("sources.json")
|
||||
self.filesURL = self.rootURL.appendingPathComponent("files", isDirectory: true)
|
||||
}
|
||||
|
||||
func load() throws -> [IngestedSource] {
|
||||
guard FileManager.default.fileExists(atPath: metadataURL.path) else { return [] }
|
||||
let data = try Data(contentsOf: metadataURL)
|
||||
return try JSONDecoder.sourceDecoder.decode([IngestedSource].self, from: data)
|
||||
.sorted { $0.createdAt > $1.createdAt }
|
||||
}
|
||||
|
||||
func save(_ sources: [IngestedSource]) throws {
|
||||
try FileManager.default.createDirectory(at: rootURL, withIntermediateDirectories: true)
|
||||
let data = try JSONEncoder.sourceEncoder.encode(sources)
|
||||
try data.write(to: metadataURL, options: [.atomic])
|
||||
}
|
||||
|
||||
func importFile(from url: URL, tags: [String]) throws -> IngestedSource {
|
||||
try FileManager.default.createDirectory(at: filesURL, withIntermediateDirectories: true)
|
||||
|
||||
let accessed = url.startAccessingSecurityScopedResource()
|
||||
defer {
|
||||
if accessed { url.stopAccessingSecurityScopedResource() }
|
||||
}
|
||||
|
||||
let id = UUID()
|
||||
let originalFilename = url.lastPathComponent
|
||||
let ext = url.pathExtension.isEmpty ? "dat" : url.pathExtension
|
||||
let localFilename = "\(id.uuidString).\(ext)"
|
||||
let destination = filesURL.appendingPathComponent(localFilename)
|
||||
if FileManager.default.fileExists(atPath: destination.path) {
|
||||
try FileManager.default.removeItem(at: destination)
|
||||
}
|
||||
try FileManager.default.copyItem(at: url, to: destination)
|
||||
|
||||
let kind = url.pathExtension.lowercased() == "pdf" ? IngestedSourceKind.pdf : .file
|
||||
let bodyText = kind == .pdf
|
||||
? PDFTextExtractor.extractText(from: destination)
|
||||
: PlainTextExtractor.extractText(from: destination)
|
||||
|
||||
return IngestedSource(
|
||||
id: id,
|
||||
kind: kind,
|
||||
title: originalFilename.removingFileExtension,
|
||||
bodyText: bodyText,
|
||||
originalFilename: originalFilename,
|
||||
localFilename: localFilename,
|
||||
tags: tags
|
||||
)
|
||||
}
|
||||
|
||||
func delete(_ source: IngestedSource) throws {
|
||||
if let url = fileURL(for: source), FileManager.default.fileExists(atPath: url.path) {
|
||||
try FileManager.default.removeItem(at: url)
|
||||
}
|
||||
var loaded = try load()
|
||||
loaded.removeAll { $0.id == source.id }
|
||||
try save(loaded)
|
||||
}
|
||||
|
||||
func fileURL(for source: IngestedSource) -> URL? {
|
||||
guard let localFilename = source.localFilename else { return nil }
|
||||
return filesURL.appendingPathComponent(localFilename)
|
||||
}
|
||||
}
|
||||
|
||||
private enum PDFTextExtractor {
|
||||
static func extractText(from url: URL) -> String {
|
||||
guard let document = PDFDocument(url: url) else { return "" }
|
||||
var pages: [String] = []
|
||||
for index in 0..<document.pageCount {
|
||||
if let text = document.page(at: index)?.string?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!text.isEmpty {
|
||||
pages.append(text)
|
||||
}
|
||||
}
|
||||
return pages.joined(separator: "\n\n")
|
||||
}
|
||||
}
|
||||
|
||||
private enum PlainTextExtractor {
|
||||
static func extractText(from url: URL) -> String {
|
||||
guard let data = try? Data(contentsOf: url) else { return "" }
|
||||
return String(data: data, encoding: .utf8)
|
||||
?? String(data: data, encoding: .ascii)
|
||||
?? ""
|
||||
}
|
||||
}
|
||||
|
||||
private extension String {
|
||||
var removingFileExtension: String {
|
||||
let ns = self as NSString
|
||||
let name = ns.deletingPathExtension
|
||||
return name.isEmpty ? self : name
|
||||
}
|
||||
}
|
||||
|
||||
private extension JSONEncoder {
|
||||
static var sourceEncoder: JSONEncoder {
|
||||
let encoder = JSONEncoder()
|
||||
encoder.dateEncodingStrategy = .iso8601
|
||||
encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
|
||||
return encoder
|
||||
}
|
||||
}
|
||||
|
||||
private extension JSONDecoder {
|
||||
static var sourceDecoder: JSONDecoder {
|
||||
let decoder = JSONDecoder()
|
||||
decoder.dateDecodingStrategy = .iso8601
|
||||
return decoder
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,8 @@ final class PodcastGenerationManager {
|
||||
func enqueue(articleUrl: String,
|
||||
title: String,
|
||||
parentBookmarkUrl: String? = nil,
|
||||
sourceText: String? = nil,
|
||||
sourceKind: String? = nil,
|
||||
onReady: ((URL, String) -> Void)? = nil) {
|
||||
let cached = ClaudeService.cachedPodcastURL(for: articleUrl)
|
||||
if FileManager.default.fileExists(atPath: cached.path) {
|
||||
@@ -52,7 +54,12 @@ final class PodcastGenerationManager {
|
||||
let task = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
do {
|
||||
let jobId = try await claude.generatePodcast(url: articleUrl)
|
||||
let jobId = try await claude.generatePodcast(
|
||||
url: articleUrl,
|
||||
title: title,
|
||||
sourceText: sourceText,
|
||||
sourceKind: sourceKind
|
||||
)
|
||||
while !Task.isCancelled {
|
||||
let status = try await claude.podcastStatus(jobId: jobId)
|
||||
update(articleUrl) {
|
||||
|
||||
40
Marks/Services/SourceSpotlightIndexer.swift
Normal file
40
Marks/Services/SourceSpotlightIndexer.swift
Normal file
@@ -0,0 +1,40 @@
|
||||
import CoreSpotlight
|
||||
import Foundation
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
enum SourceSpotlightIndexer {
|
||||
private static let domainIdentifier = "com.magicive.marks.sources"
|
||||
|
||||
static func index(_ sources: [IngestedSource]) async {
|
||||
guard CSSearchableIndex.isIndexingAvailable(), !sources.isEmpty else { return }
|
||||
let items = sources.map { source in
|
||||
let attributes = CSSearchableItemAttributeSet(contentType: source.kind == .pdf ? .pdf : .text)
|
||||
attributes.title = source.displayTitle
|
||||
attributes.contentDescription = source.bodyText
|
||||
attributes.keywords = source.tags + [source.kind.label, source.originalFilename].compactMap { $0 }
|
||||
attributes.displayName = source.originalFilename ?? source.displayTitle
|
||||
return CSSearchableItem(
|
||||
uniqueIdentifier: source.id.uuidString,
|
||||
domainIdentifier: domainIdentifier,
|
||||
attributeSet: attributes
|
||||
)
|
||||
}
|
||||
|
||||
do {
|
||||
try await CSSearchableIndex.default().indexSearchableItems(items)
|
||||
Log.spotlight.info("Indexed \(sources.count, privacy: .public) sources")
|
||||
} catch {
|
||||
Log.spotlight.error("Source index failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
static func remove(ids: [UUID]) async {
|
||||
guard CSSearchableIndex.isIndexingAvailable(), !ids.isEmpty else { return }
|
||||
do {
|
||||
try await CSSearchableIndex.default().deleteSearchableItems(withIdentifiers: ids.map(\.uuidString))
|
||||
Log.spotlight.info("Removed \(ids.count, privacy: .public) sources")
|
||||
} catch {
|
||||
Log.spotlight.error("Source delete failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user