Import portable browser profile data

cmd/browser_import_profile_data_test.go

@@ -0,0 +1,132 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	localbrowser "github.com/kernel/cli/internal/browserimport"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLocalStorageSelectionUsesAllSitesWithinLimit(t *testing.T) {
+	sites := []localbrowser.StorageSite{
+		{Origin: "https://example.com", Bytes: 1024},
+		{Origin: "https://other.example", Bytes: 2048},
+	}
+
+	selected, err := (ProfilesImportLocalCmd{}).chooseLocalStorageSites(sites, true)
+	require.NoError(t, err)
+	require.Equal(t, []string{"https://example.com", "https://other.example"}, selected)
+}
+
+func TestLocalStorageSelectionRequiresReviewWhenOverLimit(t *testing.T) {
+	sites := []localbrowser.StorageSite{{Origin: "https://large.example", Bytes: localbrowser.MaxPortableStorageSize + 1}}
+
+	_, err := (ProfilesImportLocalCmd{}).chooseLocalStorageSites(sites, true)
+	require.ErrorContains(t, err, "run interactively to choose websites")
+}
+
+func TestSelectedProfileCategoriesUsePortableApplyOrder(t *testing.T) {
+	categories := selectedProfileCategories(map[string]int{
+		"bookmarks": 2,
+		"cookies":   3,
+		"history":   4,
+		"storage":   5,
+	})
+
+	require.Equal(t, []string{"cookies", "storage", "bookmarks", "history"}, categories)
+}
+
+func TestProfilesImportLocalDefaultsHistoryOn(t *testing.T) {
+	flag := profilesImportLocalCmd.Flags().Lookup("history")
+	require.NotNil(t, flag)
+	require.Equal(t, "true", flag.DefValue)
+}
+
+func TestFitBrowserImportBundleRemovesLargestStorageOriginsFirst(t *testing.T) {
+	data := localbrowser.ProfileData{
+		Storage: []localbrowser.StorageRecord{
+			{Origin: "https://large.example", Key: "one", Value: "a much larger local storage value"},
+			{Origin: "https://small.example", Key: "two", Value: "x"},
+		},
+		History: []localbrowser.HistoryRecord{{URL: "https://example.com"}},
+	}
+	counts := map[string]int{"cookies": 2, "storage": 2, "history": 1}
+	builder := sizedBundleBuilder(50, 5, map[string]int64{
+		"https://large.example": 10,
+		"https://small.example": 2,
+	})
+
+	result, err := fitBrowserImportBundle(data, counts, builder)
+	require.NoError(t, err)
+	require.Equal(t, []string{"https://large.example"}, result.skippedStorageOrigins)
+	require.Equal(t, 1, result.skippedStorageRecords)
+	require.Zero(t, result.skippedHistoryRecords)
+	require.Len(t, result.data.Storage, 1)
+	require.Equal(t, "https://small.example", result.data.Storage[0].Origin)
+	require.Equal(t, 1, result.itemCounts["storage"])
+	require.Equal(t, 1, result.itemCounts["history"])
+}
+
+func TestFitBrowserImportBundleDropsHistoryThenRestoresStorageThatFits(t *testing.T) {
+	data := localbrowser.ProfileData{
+		Storage: []localbrowser.StorageRecord{
+			{Origin: "https://large.example", Key: "one", Value: "a much larger local storage value"},
+			{Origin: "https://small.example", Key: "two", Value: "x"},
+		},
+		History: []localbrowser.HistoryRecord{{URL: "https://example.com"}, {URL: "https://other.example"}},
+	}
+	counts := map[string]int{"cookies": 2, "storage": 2, "history": 2}
+	builder := sizedBundleBuilder(60, 10, map[string]int64{
+		"https://large.example": 3,
+		"https://small.example": 2,
+	})
+
+	result, err := fitBrowserImportBundle(data, counts, builder)
+	require.NoError(t, err)
+	require.Equal(t, []string{"https://large.example"}, result.skippedStorageOrigins)
+	require.Equal(t, 1, result.skippedStorageRecords)
+	require.Equal(t, 2, result.skippedHistoryRecords)
+	require.Len(t, result.data.Storage, 1)
+	require.Empty(t, result.data.History)
+	require.NotContains(t, result.itemCounts, "history")
+}
+
+func TestFitBrowserImportBundleLeavesBundleAloneWhenItFits(t *testing.T) {
+	data := localbrowser.ProfileData{Storage: []localbrowser.StorageRecord{{Origin: "https://example.com", Key: "one", Value: "value"}}}
+	counts := map[string]int{"cookies": 2, "storage": 1}
+
+	result, err := fitBrowserImportBundle(data, counts, sizedBundleBuilder(60, 0, map[string]int64{"https://example.com": 3}))
+	require.NoError(t, err)
+	require.Zero(t, result.originalSize)
+	require.Equal(t, data.Storage, result.data.Storage)
+	require.Equal(t, counts, result.itemCounts)
+}
+
+func TestFitBrowserImportBundleRejectsOversizedRequiredData(t *testing.T) {
+	_, err := fitBrowserImportBundle(localbrowser.ProfileData{}, map[string]int{"cookies": 2}, sizedBundleBuilder(65, 0, nil))
+	require.ErrorContains(t, err, "cookies and bookmarks do not fit")
+	require.True(t, errors.Is(err, localbrowser.ErrBundleTooLarge))
+}
+
+func sizedBundleBuilder(base, history int64, storage map[string]int64) profileBundleBuilder {
+	return func(data localbrowser.ProfileData) ([]byte, error) {
+		size := base
+		if len(data.History) > 0 {
+			size += history
+		}
+		seen := make(map[string]struct{})
+		for _, record := range data.Storage {
+			if _, ok := seen[record.Origin]; ok {
+				continue
+			}
+			seen[record.Origin] = struct{}{}
+			size += storage[record.Origin]
+		}
+		if size > 64 {
+			return nil, &localbrowser.BundleTooLargeError{Size: size, Limit: 64}
+		}
+		return []byte(fmt.Sprintf("%d", size)), nil
+	}
+}

cmd/connector.go

@@ -59,6 +59,7 @@ func runConnectorOpen(cmd *cobra.Command, args []string) error {
 		Version:         metadata.Version,
 		WaitTimeout:     30 * time.Minute,
 		DashboardLaunch: true,
+		ImportHistory:   true,
 	}
 	project, err := validateConnectorProject(cmd, input.ProjectID)
 	if err != nil {

cmd/profiles_import_local.go

@@ -41,6 +41,7 @@ type ProfilesImportLocalInput struct {
 	PasswordManager    string
 	InstallAgentSkills bool
 	DashboardLaunch    bool
+	ImportHistory      bool
 	Project            *kernel.Project
 }
 
@@ -185,6 +186,11 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI
 	if len(cookieSelection.sites) == 0 {
 		return fmt.Errorf("select at least one website")
 	}
+	since := c.now().AddDate(0, 0, -in.Days)
+	profileDataSelection, err := c.chooseLocalProfileData(ctx, profile, since, in.ImportHistory, nonInteractive, humanOutput)
+	if err != nil {
+		return err
+	}
 	pendingLogins := pendingManagedAuth{}
 	if managedAuthImportRequested(in.PasswordManager, nonInteractive) {
 		phaseStarted = time.Now()
@@ -196,6 +202,16 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI
 			return err
 		}
 	}
+	if !nonInteractive {
+		proceed, err := c.confirmBrowserImport(targetName, cookieSelection, cookieSites, profileDataSelection, pendingLogins)
+		if err != nil {
+			return err
+		}
+		if !proceed {
+			pterm.Info.Println("Browser import canceled; no Kernel resources were changed")
+			return nil
+		}
+	}
 	if humanOutput {
 		pterm.Println()
 		if cookieSelection.all {
@@ -224,11 +240,34 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI
 		version = "dev"
 	}
 	phaseStarted = time.Now()
-	bundle, err := localbrowser.BuildCookieBundle(ctx, profile, targetName, version, cookies)
+	profileData, itemCounts, err := buildSelectedProfileData(ctx, profile, profileDataSelection, cookies, since)
+	if err != nil {
+		return err
+	}
+	fit, err := fitBrowserImportBundle(profileData, itemCounts, func(candidate localbrowser.ProfileData) ([]byte, error) {
+		return localbrowser.BuildProfileBundle(ctx, profile, targetName, version, candidate)
+	})
 	timings["bundle"] = time.Since(phaseStarted)
 	if err != nil {
 		return err
 	}
+	if fit.originalSize > 0 {
+		if nonInteractive {
+			return fmt.Errorf("%w; run interactively to review optional browser data that can be skipped", &localbrowser.BundleTooLargeError{Size: fit.originalSize, Limit: fit.limit})
+		}
+		proceed, err := c.confirmBundleFallback(fit)
+		if err != nil {
+			return err
+		}
+		if !proceed {
+			pterm.Info.Println("Browser import canceled; no Kernel resources were changed")
+			return nil
+		}
+	}
+	profileData = fit.data
+	itemCounts = fit.itemCounts
+	bundle := fit.bundle
+	categories := selectedProfileCategories(itemCounts)
 	token, err := auth.BearerToken(ctx)
 	if err != nil {
 		return err
@@ -247,13 +286,13 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI
 	}
 	inventory := localbrowser.Inventory{Sources: []localbrowser.Source{{
 		ID: profile.ID, Kind: "browser", Name: profile.DisplayName(), Browser: profile.Browser.ID,
-		DataTypes: []string{"cookies"}, ItemCounts: map[string]int{"cookies": len(cookies)},
+		DataTypes: categories, ItemCounts: itemCounts,
 	}}}
 	status, err := client.SubmitInventory(ctx, created.ID, created.HelperToken, inventory)
 	if err != nil {
 		return browserImportProgressError(created.ID, status.Phase, time.Since(phaseStarted), err)
 	}
-	selection := localbrowser.Selection{Profiles: []localbrowser.ProfileSelection{{SourceID: profile.ID, TargetName: targetName, Categories: []string{"cookies"}}}, CredentialSources: make([]string, 0)}
+	selection := localbrowser.Selection{Profiles: []localbrowser.ProfileSelection{{SourceID: profile.ID, TargetName: targetName, Categories: categories}}, CredentialSources: make([]string, 0)}
 	status, err = client.SubmitSelection(ctx, created.ID, selection)
 	if err != nil {
 		return browserImportProgressError(created.ID, status.Phase, time.Since(phaseStarted), err)
@@ -275,6 +314,15 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI
 	profileID := status.Applied.Profiles[0].ProfileID
 	if humanOutput {
 		pterm.Success.Printf("Imported %d cookies from %d websites\n", len(cookies), importedCookieSites)
+		if count := itemCounts["bookmarks"]; count > 0 {
+			pterm.Success.Printf("Imported %d bookmarks\n", count)
+		}
+		if count := itemCounts["history"]; count > 0 {
+			pterm.Success.Printf("Imported %d history entries\n", count)
+		}
+		if count := itemCounts["storage"]; count > 0 {
+			pterm.Success.Printf("Imported %d local storage keys from %d origins\n", count, importedStorageOriginCount(profileData.Storage))
+		}
 	}
 	connectionIDs := make([]string, 0)
 	approvedLogins := make([]passwordmanager.Record, 0)
@@ -325,7 +373,7 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI
 		}
 	}
 	if in.Output == "json" {
-		data, err := json.MarshalIndent(map[string]any{"profile_id": profileID, "profile_name": targetName, "sites": cookieSelection.sites, "cookies_imported": len(cookies), "managed_auth_connections": connectionIDs, "agent_skills_installed": installedSkills, "agent_skill_warning": skillWarning, "duration_ms": time.Since(startedAt).Milliseconds(), "timings_ms": durationMilliseconds(timings)}, "", "  ")
+		data, err := json.MarshalIndent(map[string]any{"profile_id": profileID, "profile_name": targetName, "sites": cookieSelection.sites, "cookies_imported": itemCounts["cookies"], "browser_data_imported": itemCounts, "managed_auth_connections": connectionIDs, "agent_skills_installed": installedSkills, "agent_skill_warning": skillWarning, "duration_ms": time.Since(startedAt).Milliseconds(), "timings_ms": durationMilliseconds(timings)}, "", "  ")
 		if err != nil {
 			return err
 		}
@@ -1405,8 +1453,8 @@ var cuidLikeProfileName = regexp.MustCompile(`^[a-z0-9]{24}$`)
 
 var profilesImportLocalCmd = &cobra.Command{
 	Use:   "import-local",
-	Short: "Import cookies from a local browser",
-	Long:  "Import all cookies or selected websites from a local Google Chrome or Helium profile on macOS into a Kernel browser profile.",
+	Short: "Import a local browser profile",
+	Long:  "Import cookies and selected portable data from a local Google Chrome or Helium profile on macOS into a Kernel browser profile.",
 	Args:  cobra.NoArgs,
 	RunE:  runProfilesImportLocal,
 }
@@ -1427,6 +1475,7 @@ func init() {
 	profilesImportLocalCmd.Flags().Int("days", 30, "Rank websites used during the last number of days (1-90)")
 	profilesImportLocalCmd.Flags().Duration("wait-timeout", 30*time.Minute, "Maximum time to wait for the import to complete")
 	profilesImportLocalCmd.Flags().BoolP("yes", "y", false, "Import all cookies and use unambiguous defaults without prompting")
+	profilesImportLocalCmd.Flags().Bool("history", true, "Include browsing history from the selected --days window")
 	profilesImportLocalCmd.Flags().String("password-manager", "", "Password managers to search: bitwarden, 1password, both comma-separated, all, or none")
 	profilesImportLocalCmd.Flags().Bool("install-agent-skills", false, "Install the Kernel Managed Auth skill into detected agent directories")
 	addJSONOutputFlag(profilesImportLocalCmd)
@@ -1442,12 +1491,13 @@ func runProfilesImportLocal(cmd *cobra.Command, _ []string) error {
 	skipConfirm, _ := cmd.Flags().GetBool("yes")
 	passwordManager, _ := cmd.Flags().GetString("password-manager")
 	installAgentSkills, _ := cmd.Flags().GetBool("install-agent-skills")
+	importHistory, _ := cmd.Flags().GetBool("history")
 	output, _ := cmd.Flags().GetString("output")
 	project, _ := cmd.Flags().GetString("project")
 	return runProfilesImportLocalWithInput(cmd, ProfilesImportLocalInput{
 		BrowserProfile: browserProfile, ProfileName: profileName, Sites: sites, Days: days,
 		SkipConfirm: skipConfirm, Output: output, ProjectID: resolveProjectSelection(project), Version: metadata.Version,
-		WaitTimeout: waitTimeout, PasswordManager: passwordManager, InstallAgentSkills: installAgentSkills,
+		WaitTimeout: waitTimeout, PasswordManager: passwordManager, InstallAgentSkills: installAgentSkills, ImportHistory: importHistory,
 	})
 }
 

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/pflag v1.0.6
 	github.com/stretchr/testify v1.11.1
+	github.com/syndtr/goleveldb v1.0.0
 	github.com/zalando/go-keyring v0.2.6
 	golang.org/x/crypto v0.52.0
 	golang.org/x/net v0.54.0
@@ -40,6 +41,7 @@ require (
 	github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
 	github.com/gookit/color v1.5.4 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect